@SuppressWarnings("unchecked") @Override public VectorWrapper<T> cloneAndTransfer(BufferAllocator allocator, CallBack callback) { try{ TransferPair tp = vector.getTransferPair(vector.getField().getName(), allocator, callback); tp.transfer(); return new SimpleVectorWrapper<T>((T) tp.getTo()); }catch(RuntimeException ex){ throw ex; } }
@Override public void splitAndTransfer(int startIndex, int length) { for (TransferPair p : pairs) { p.splitAndTransfer(startIndex, length); } to.setValueCount(length); } }
@Override public void copyValueSafe(int from, int to) { for (TransferPair p : pairs) { p.copyValueSafe(from, to); } }
void transferOut() { for(TransferPair transferPair : transferPairList) { transferPair.transfer(); } }
@Override @SuppressWarnings("unchecked") public VectorWrapper<T> cloneAndTransfer(BufferAllocator allocator, CallBack callback) { if(vectors.size() == 0){ return new HyperVectorWrapper<T>(field, (T[])vectors.toArray(), false); }else { T[] newVectors = (T[]) Array.newInstance(vectors.get(0).getClass(), vectors.size()); for(int i =0; i < vectors.size(); i++){ newVectors[i] = (T) vectors.get(i).getTransferPair(vectors.get(i).getField().getName(), allocator, callback).getTo(); } return new HyperVectorWrapper<T>(field, (T[])vectors.toArray(), false); } }
/** * Decodes a dictionary encoded array using the provided dictionary. * * @param indices dictionary encoded values, must be int type * @param dictionary dictionary used to decode the values * @return vector with values restored from dictionary */ public static ValueVector decode(ValueVector indices, Dictionary dictionary) { int count = indices.getValueCount(); ValueVector dictionaryVector = dictionary.getVector(); int dictionaryCount = dictionaryVector.getValueCount(); // copy the dictionary values into the decoded vector TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator()); transfer.getTo().allocateNewSafe(); for (int i = 0; i < count; i++) { Object index = indices.getObject(i); if (index != null) { int indexAsInt = ((Number) index).intValue(); if (indexAsInt > dictionaryCount) { throw new IllegalArgumentException("Provided dictionary does not contain value for index " + indexAsInt); } transfer.copyValueSafe(indexAsInt, i); } } // TODO do we need to worry about the field? ValueVector decoded = transfer.getTo(); decoded.setValueCount(count); return decoded; }
final VarCharVector newVarCharVector = (VarCharVector) tp.getTo(); final int[][] startLengths = {{0, 201}, {201, 200}, {401, 99}}; final int start = startLength[0]; final int length = startLength[1]; tp.splitAndTransfer(start, length); newVarCharVector.setValueCount(length); for (int i = 0; i < length; i++) {
private void doTransfers(){ for(TransferPair t : tx){ t.transfer(); } }
@Override public VectorAccessible setup(VectorAccessible accessible) throws Exception { state.is(State.NEEDS_SETUP); List<TransferPair> pairs = new ArrayList<>(); for(VectorWrapper<?> w : accessible){ TransferPair pair = w.getValueVector().getTransferPair(context.getAllocator()); pairs.add(pair); outgoing.add(pair.getTo()); } transfers = ImmutableList.copyOf(pairs); outgoing.buildSchema(SelectionVectorMode.NONE); outgoing.setInitialCapacity(context.getTargetBatchSize()); state = State.CAN_CONSUME; return outgoing; }
@Override public int outputData() throws Exception { state.is(State.CAN_PRODUCE); final int records = incoming.getRecordCount(); for(JsonConverter<?> converter : converters){ converter.convert(records); } for(TransferPair transfer : transfers){ transfer.transfer(); transfer.getTo().setValueCount(records); } outgoing.setRecordCount(records); state = State.CAN_CONSUME; return records; }
@Override public void transfer() { to.clear(); to.typeBuffer = typeBuffer.transferOwnership(to.allocator).buffer; internalStructVectorTransferPair.transfer(); to.valueCount = valueCount; clear(); }
@Override public VectorAccessible setup(VectorAccessible left, VectorAccessible right) throws Exception { state.is(State.NEEDS_SETUP); List<Field> leftFields = left.getSchema().getFields(); List<Field> rightFields = right.getSchema().getFields(); Preconditions.checkArgument(leftFields.size() == rightFields.size(), "Field counts don't match. Left: \n%s\nRight:\n%s", left.getSchema(), right.getSchema()); for(int i =0; i < leftFields.size(); i++){ Field leftField = leftFields.get(i); Field rightField = rightFields.get(i); Preconditions.checkArgument(leftField.getType().equals(rightField.getType()), "Field types don't match. Left: \n%s\nRight:\n%s", left.getSchema(), right.getSchema()); } List<TransferPair> leftTransfers = new ArrayList<>(); List<TransferPair> rightTransfers = new ArrayList<>(); final Iterator<VectorWrapper<?>> rightIter = right.iterator(); for(VectorWrapper<?> leftWrapper : left){ ValueVector vector = leftWrapper.getValueVector(); TransferPair pair = vector.getTransferPair(context.getAllocator()); leftTransfers.add(pair); outgoing.add(pair.getTo()); assert rightIter.hasNext(); ValueVector rightVector = rightIter.next().getValueVector(); rightTransfers.add(rightVector.makeTransferPair(pair.getTo())); } this.leftTransfers = ImmutableList.copyOf(leftTransfers); this.rightTransfers = ImmutableList.copyOf(rightTransfers); outgoing.buildSchema(SelectionVectorMode.NONE); state = State.CAN_CONSUME_L; return outgoing; }
@Override public void splitAndTransfer(int startIndex, int length) { to.clear(); internalStructVectorTransferPair.splitAndTransfer(startIndex, length); final int startPoint = startIndex * TYPE_WIDTH; final int sliceLength = length * TYPE_WIDTH; to.typeBuffer = typeBuffer.slice(startPoint, sliceLength).transferOwnership(to.allocator).buffer; to.setValueCount(length); }
@Override public void copy(long compoundAddr, final int srcStart, int count) { copyWatch.start(); final long max = compoundAddr + count * OFFSET_SIZE; for(int from = srcStart; compoundAddr < max; compoundAddr +=OFFSET_SIZE, from++) { final int compoundIdx = PlatformDependent.getInt(compoundAddr); final int batchIdx = compoundIdx >>> 16; final int rowIdx = compoundIdx & 65535; transfers[batchIdx].copyValueSafe(from, rowIdx); } copyWatch.stop(); } }
private VectorContainer transferBatch(VectorAccessible batch) { @SuppressWarnings("resource") // TODO better way to write this? VectorContainer container = new VectorContainer(); final List<ValueVector> vectors = Lists.newArrayList(); for (VectorWrapper<?> v : batch) { if (v.isHyper()) { throw new UnsupportedOperationException("Record batch data can't be created based on a hyper batch."); } TransferPair tp = v.getValueVector().getTransferPair(allocator); tp.transfer(); vectors.add(tp.getTo()); } container.addCollection(vectors); container.setRecordCount(batch.getRecordCount()); container.buildSchema(SelectionVectorMode.NONE); return container; }
@Override public void output(int batchIndex) { pairs[batchIndex].transfer(); }
FieldVector vector = (FieldVector) output.getTransferPair(output.getAllocator()).getTo(); TransferPair tp = vector.makeTransferPair(output); ((FixedWidthVector) vector).allocateNew(LBlockHashTableNoSpill.MAX_VALUES_PER_BATCH);
t.splitAndTransfer(initialInnerValueIndex, delta);
public void copyFrom(int fromIndex, int thisIndex, FixedSizeListVector from) { TransferPair pair = from.makeTransferPair(this); pair.copyValueSafe(fromIndex, thisIndex); }
private void loadNextBatch(boolean first) throws IOException{ Preconditions.checkArgument(batchIndex + 1 < batchIndexMax, "You tried to go beyond end of available batches to read."); container.zeroVectors(); /* uncompress the data when de-serializing the spilled data into ArrowBufs */ final VectorAccessibleSerializable serializer = new VectorAccessibleSerializable(allocator, compressSpilledBatch, compressSpilledBatchAllocator); serializer.readFromStream(inputStream); final VectorContainer incoming = serializer.get(); Iterator<VectorWrapper<?>> wrapperIterator = incoming.iterator(); if(first){ for(VectorWrapper<?> wrapper : incoming){ final ValueVector sourceVector = wrapper.getValueVector(); TransferPair pair = sourceVector.getTransferPair(allocator); final ValueVector targetVector = pair.getTo(); pair.transfer(); container.add(targetVector); } container.buildSchema(); }else{ for (VectorWrapper<?> w : container) { final ValueVector sourceVector = wrapperIterator.next().getValueVector(); final TransferPair pair = sourceVector.makeTransferPair(w.getValueVector()); pair.transfer(); } } recordIndexMax = incoming.getRecordCount(); batchIndex++; recordIndex = -1; }