@Override @SuppressWarnings("unchecked") public VectorWrapper<T> cloneAndTransfer(BufferAllocator allocator, CallBack callback) { if(vectors.size() == 0){ return new HyperVectorWrapper<T>(field, (T[])vectors.toArray(), false); }else { T[] newVectors = (T[]) Array.newInstance(vectors.get(0).getClass(), vectors.size()); for(int i =0; i < vectors.size(); i++){ newVectors[i] = (T) vectors.get(i).getTransferPair(vectors.get(i).getField().getName(), allocator, callback).getTo(); } return new HyperVectorWrapper<T>(field, (T[])vectors.toArray(), false); } }
@Override public VectorAccessible setup(VectorAccessible accessible) throws Exception { state.is(State.NEEDS_SETUP); List<TransferPair> pairs = new ArrayList<>(); for(VectorWrapper<?> w : accessible){ TransferPair pair = w.getValueVector().getTransferPair(context.getAllocator()); pairs.add(pair); outgoing.add(pair.getTo()); } transfers = ImmutableList.copyOf(pairs); outgoing.buildSchema(SelectionVectorMode.NONE); outgoing.setInitialCapacity(context.getTargetBatchSize()); state = State.CAN_CONSUME; return outgoing; }
@Override public VectorAccessible setup(VectorAccessible left, VectorAccessible right) throws Exception { state.is(State.NEEDS_SETUP); List<Field> leftFields = left.getSchema().getFields(); List<Field> rightFields = right.getSchema().getFields(); Preconditions.checkArgument(leftFields.size() == rightFields.size(), "Field counts don't match. Left: \n%s\nRight:\n%s", left.getSchema(), right.getSchema()); for(int i =0; i < leftFields.size(); i++){ Field leftField = leftFields.get(i); Field rightField = rightFields.get(i); Preconditions.checkArgument(leftField.getType().equals(rightField.getType()), "Field types don't match. Left: \n%s\nRight:\n%s", left.getSchema(), right.getSchema()); } List<TransferPair> leftTransfers = new ArrayList<>(); List<TransferPair> rightTransfers = new ArrayList<>(); final Iterator<VectorWrapper<?>> rightIter = right.iterator(); for(VectorWrapper<?> leftWrapper : left){ ValueVector vector = leftWrapper.getValueVector(); TransferPair pair = vector.getTransferPair(context.getAllocator()); leftTransfers.add(pair); outgoing.add(pair.getTo()); assert rightIter.hasNext(); ValueVector rightVector = rightIter.next().getValueVector(); rightTransfers.add(rightVector.makeTransferPair(pair.getTo())); } this.leftTransfers = ImmutableList.copyOf(leftTransfers); this.rightTransfers = ImmutableList.copyOf(rightTransfers); outgoing.buildSchema(SelectionVectorMode.NONE); state = State.CAN_CONSUME_L; return outgoing; }
private VectorContainer transferBatch(VectorAccessible batch) { @SuppressWarnings("resource") // TODO better way to write this? VectorContainer container = new VectorContainer(); final List<ValueVector> vectors = Lists.newArrayList(); for (VectorWrapper<?> v : batch) { if (v.isHyper()) { throw new UnsupportedOperationException("Record batch data can't be created based on a hyper batch."); } TransferPair tp = v.getValueVector().getTransferPair(allocator); tp.transfer(); vectors.add(tp.getTo()); } container.addCollection(vectors); container.setRecordCount(batch.getRecordCount()); container.buildSchema(SelectionVectorMode.NONE); return container; }
@SuppressWarnings("unchecked") @Override public VectorWrapper<T> cloneAndTransfer(BufferAllocator allocator, CallBack callback) { try{ TransferPair tp = vector.getTransferPair(vector.getField().getName(), allocator, callback); tp.transfer(); return new SimpleVectorWrapper<T>((T) tp.getTo()); }catch(RuntimeException ex){ throw ex; } }
TransferPair pair = incomingVector.getTransferPair(context.getAllocator()); transfers.add(pair); outgoing.add(pair.getTo());
public RecordBatchData(VectorAccessible batch, BufferAllocator allocator, boolean applySVMode) { this.recordCount = batch.getRecordCount(); final List<ValueVector> vectors = Lists.newArrayList(); if (applySVMode && batch.getSchema().getSelectionVectorMode() == SelectionVectorMode.TWO_BYTE) { this.sv2 = batch.getSelectionVector2().clone(); } else { this.sv2 = null; } for (VectorWrapper<?> v : batch) { if (v.isHyper()) { throw new UnsupportedOperationException("Record batch data can't be created based on a hyper batch."); } TransferPair tp = v.getValueVector().getTransferPair(allocator); tp.transfer(); vectors.add(tp.getTo()); } container.addCollection(vectors); container.setRecordCount(recordCount); container.buildSchema(applySVMode ? batch.getSchema().getSelectionVectorMode() : SelectionVectorMode.NONE); }
/** * Decodes a dictionary encoded array using the provided dictionary. * * @param indices dictionary encoded values, must be int type * @param dictionary dictionary used to decode the values * @return vector with values restored from dictionary */ public static ValueVector decode(ValueVector indices, Dictionary dictionary) { int count = indices.getValueCount(); ValueVector dictionaryVector = dictionary.getVector(); int dictionaryCount = dictionaryVector.getValueCount(); // copy the dictionary values into the decoded vector TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator()); transfer.getTo().allocateNewSafe(); for (int i = 0; i < count; i++) { Object index = indices.getObject(i); if (index != null) { int indexAsInt = ((Number) index).intValue(); if (indexAsInt > dictionaryCount) { throw new IllegalArgumentException("Provided dictionary does not contain value for index " + indexAsInt); } transfer.copyValueSafe(indexAsInt, i); } } // TODO do we need to worry about the field? ValueVector decoded = transfer.getTo(); decoded.setValueCount(count); return decoded; }
for (final VectorWrapper<?> wrapper : batch) { ValueVector vector = wrapper.getValueVector(); TransferPair pair = vector.getTransferPair(context.getAllocator()); outgoing.add(pair.getTo()); transfers.add(pair);
/** * The data layout is the same for the actual data within a repeated field, as it is in a scalar vector for * the same sql type. For example, a repeated int vector has a vector of offsets into a regular int vector to * represent the lists. As the data layout for the actual values in the same in the repeated vector as in the * scalar vector of the same type, we can avoid making individual copies for the column being flattened, and just * use vector copies between the inner vector of the repeated field to the resulting scalar vector from the flatten * operation. This is completed after we determine how many records will fit (as we will hit either a batch end, or * the end of one of the other vectors while we are copying the data of the other vectors alongside each new flattened * value coming out of the repeated field.) */ private TransferPair getFlattenFieldTransferPair(FieldReference outputName) { final TypedFieldId fieldId = incoming.getSchema().getFieldId(config.getColumn()); final Class<? extends ValueVector> vectorClass = TypeHelper.getValueVectorClass(incoming.getSchema().getColumn(fieldId.getFieldIds()[0])); final ValueVector flattenField = incoming.getValueAccessorById(vectorClass, fieldId.getFieldIds()).getValueVector(); final ValueVector vvIn = RepeatedValueVector.class.cast(flattenField).getDataVector(); return vvIn.getTransferPair(outputName.getAsNamePart().getName(), context.getAllocator()); }
private void loadNextBatch(boolean first) throws IOException{ Preconditions.checkArgument(batchIndex + 1 < batchIndexMax, "You tried to go beyond end of available batches to read."); container.zeroVectors(); /* uncompress the data when de-serializing the spilled data into ArrowBufs */ final VectorAccessibleSerializable serializer = new VectorAccessibleSerializable(allocator, compressSpilledBatch, compressSpilledBatchAllocator); serializer.readFromStream(inputStream); final VectorContainer incoming = serializer.get(); Iterator<VectorWrapper<?>> wrapperIterator = incoming.iterator(); if(first){ for(VectorWrapper<?> wrapper : incoming){ final ValueVector sourceVector = wrapper.getValueVector(); TransferPair pair = sourceVector.getTransferPair(allocator); final ValueVector targetVector = pair.getTo(); pair.transfer(); container.add(targetVector); } container.buildSchema(); }else{ for (VectorWrapper<?> w : container) { final ValueVector sourceVector = wrapperIterator.next().getValueVector(); final TransferPair pair = sourceVector.makeTransferPair(w.getValueVector()); pair.transfer(); } } recordIndexMax = incoming.getRecordCount(); batchIndex++; recordIndex = -1; }
private FieldWriter promoteToUnion() { String name = vector.getField().getName(); TransferPair tp = vector.getTransferPair(vector.getMinorType().name().toLowerCase(), vector.getAllocator()); tp.transfer(); if (parentContainer != null) { // TODO allow dictionaries in complex types unionVector = parentContainer.addOrGetUnion(name); unionVector.allocateNew(); } else if (listVector != null) { unionVector = listVector.promoteToUnion(); } unionVector.addVector((FieldVector) tp.getTo()); writer = new UnionWriter(unionVector, nullableStructWriterFactory); writer.setPosition(idx()); for (int i = 0; i <= idx(); i++) { unionVector.setType(i, vector.getMinorType()); } vector = null; state = State.UNION; return writer; }
TransferPair pair = w.getValueVector().getTransferPair(context.getAllocator()); atBatInput.add(pair.getTo()); transfers.add(pair);