congrats Icon
New! Tabnine Pro 14-day free trial
Start a free trial
Tabnine Logo
ValueVector.getTransferPair
Code IndexAdd Tabnine to your IDE (free)

How to use
getTransferPair
method
in
org.apache.arrow.vector.ValueVector

Best Java code snippets using org.apache.arrow.vector.ValueVector.getTransferPair (Showing top 13 results out of 315)

origin: dremio/dremio-oss

@Override
@SuppressWarnings("unchecked")
public VectorWrapper<T> cloneAndTransfer(BufferAllocator allocator, CallBack callback) {
 if(vectors.size() == 0){
  return new HyperVectorWrapper<T>(field, (T[])vectors.toArray(), false);
 }else {
  T[] newVectors = (T[]) Array.newInstance(vectors.get(0).getClass(), vectors.size());
  for(int i =0; i < vectors.size(); i++){
   newVectors[i] = (T) vectors.get(i).getTransferPair(vectors.get(i).getField().getName(), allocator, callback).getTo();
  }
  return new HyperVectorWrapper<T>(field, (T[])vectors.toArray(), false);
 }
}
origin: dremio/dremio-oss

@Override
public VectorAccessible setup(VectorAccessible accessible) throws Exception {
 state.is(State.NEEDS_SETUP);
 List<TransferPair> pairs = new ArrayList<>();
 for(VectorWrapper<?> w : accessible){
  TransferPair pair = w.getValueVector().getTransferPair(context.getAllocator());
  pairs.add(pair);
  outgoing.add(pair.getTo());
 }
 transfers = ImmutableList.copyOf(pairs);
 outgoing.buildSchema(SelectionVectorMode.NONE);
 outgoing.setInitialCapacity(context.getTargetBatchSize());
 state = State.CAN_CONSUME;
 return outgoing;
}
origin: dremio/dremio-oss

@Override
public VectorAccessible setup(VectorAccessible left, VectorAccessible right) throws Exception {
 state.is(State.NEEDS_SETUP);
 List<Field> leftFields = left.getSchema().getFields();
 List<Field> rightFields = right.getSchema().getFields();
 Preconditions.checkArgument(leftFields.size() == rightFields.size(), "Field counts don't match. Left: \n%s\nRight:\n%s", left.getSchema(), right.getSchema());
 for(int i =0; i < leftFields.size(); i++){
  Field leftField = leftFields.get(i);
  Field rightField = rightFields.get(i);
  Preconditions.checkArgument(leftField.getType().equals(rightField.getType()), "Field types don't match. Left: \n%s\nRight:\n%s", left.getSchema(), right.getSchema());
 }
 List<TransferPair> leftTransfers = new ArrayList<>();
 List<TransferPair> rightTransfers = new ArrayList<>();
 final Iterator<VectorWrapper<?>> rightIter = right.iterator();
 for(VectorWrapper<?> leftWrapper : left){
  ValueVector vector = leftWrapper.getValueVector();
  TransferPair pair = vector.getTransferPair(context.getAllocator());
  leftTransfers.add(pair);
  outgoing.add(pair.getTo());
  assert rightIter.hasNext();
  ValueVector rightVector = rightIter.next().getValueVector();
  rightTransfers.add(rightVector.makeTransferPair(pair.getTo()));
 }
 this.leftTransfers = ImmutableList.copyOf(leftTransfers);
 this.rightTransfers = ImmutableList.copyOf(rightTransfers);
 outgoing.buildSchema(SelectionVectorMode.NONE);
 state = State.CAN_CONSUME_L;
 return outgoing;
}
origin: dremio/dremio-oss

private VectorContainer transferBatch(VectorAccessible batch) {
 @SuppressWarnings("resource") // TODO better way to write this?
 VectorContainer container = new VectorContainer();
 final List<ValueVector> vectors = Lists.newArrayList();
 for (VectorWrapper<?> v : batch) {
  if (v.isHyper()) {
   throw new UnsupportedOperationException("Record batch data can't be created based on a hyper batch.");
  }
  TransferPair tp = v.getValueVector().getTransferPair(allocator);
  tp.transfer();
  vectors.add(tp.getTo());
 }
 container.addCollection(vectors);
 container.setRecordCount(batch.getRecordCount());
 container.buildSchema(SelectionVectorMode.NONE);
 return container;
}
origin: dremio/dremio-oss

@SuppressWarnings("unchecked")
@Override
public VectorWrapper<T> cloneAndTransfer(BufferAllocator allocator, CallBack callback) {
 try{
 TransferPair tp = vector.getTransferPair(vector.getField().getName(), allocator, callback);
 tp.transfer();
 return new SimpleVectorWrapper<T>((T) tp.getTo());
 }catch(RuntimeException ex){
  throw ex;
 }
}
origin: dremio/dremio-oss

TransferPair pair = incomingVector.getTransferPair(context.getAllocator());
transfers.add(pair);
outgoing.add(pair.getTo());
origin: dremio/dremio-oss

public RecordBatchData(VectorAccessible batch, BufferAllocator allocator, boolean applySVMode) {
 this.recordCount = batch.getRecordCount();
 final List<ValueVector> vectors = Lists.newArrayList();
 if (applySVMode && batch.getSchema().getSelectionVectorMode() == SelectionVectorMode.TWO_BYTE) {
  this.sv2 = batch.getSelectionVector2().clone();
 } else {
  this.sv2 = null;
 }
 for (VectorWrapper<?> v : batch) {
  if (v.isHyper()) {
   throw new UnsupportedOperationException("Record batch data can't be created based on a hyper batch.");
  }
  TransferPair tp = v.getValueVector().getTransferPair(allocator);
  tp.transfer();
  vectors.add(tp.getTo());
 }
 container.addCollection(vectors);
 container.setRecordCount(recordCount);
 container.buildSchema(applySVMode ? batch.getSchema().getSelectionVectorMode() : SelectionVectorMode.NONE);
}
origin: org.apache.arrow/arrow-vector

/**
 * Decodes a dictionary encoded array using the provided dictionary.
 *
 * @param indices    dictionary encoded values, must be int type
 * @param dictionary dictionary used to decode the values
 * @return vector with values restored from dictionary
 */
public static ValueVector decode(ValueVector indices, Dictionary dictionary) {
 int count = indices.getValueCount();
 ValueVector dictionaryVector = dictionary.getVector();
 int dictionaryCount = dictionaryVector.getValueCount();
 // copy the dictionary values into the decoded vector
 TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator());
 transfer.getTo().allocateNewSafe();
 for (int i = 0; i < count; i++) {
  Object index = indices.getObject(i);
  if (index != null) {
   int indexAsInt = ((Number) index).intValue();
   if (indexAsInt > dictionaryCount) {
    throw new IllegalArgumentException("Provided dictionary does not contain value for index " + indexAsInt);
   }
   transfer.copyValueSafe(indexAsInt, i);
  }
 }
 // TODO do we need to worry about the field?
 ValueVector decoded = transfer.getTo();
 decoded.setValueCount(count);
 return decoded;
}
origin: dremio/dremio-oss

for (final VectorWrapper<?> wrapper : batch) {
 ValueVector vector = wrapper.getValueVector();
 TransferPair pair = vector.getTransferPair(context.getAllocator());
 outgoing.add(pair.getTo());
 transfers.add(pair);
origin: dremio/dremio-oss

/**
 * The data layout is the same for the actual data within a repeated field, as it is in a scalar vector for
 * the same sql type. For example, a repeated int vector has a vector of offsets into a regular int vector to
 * represent the lists. As the data layout for the actual values in the same in the repeated vector as in the
 * scalar vector of the same type, we can avoid making individual copies for the column being flattened, and just
 * use vector copies between the inner vector of the repeated field to the resulting scalar vector from the flatten
 * operation. This is completed after we determine how many records will fit (as we will hit either a batch end, or
 * the end of one of the other vectors while we are copying the data of the other vectors alongside each new flattened
 * value coming out of the repeated field.)
 */
private TransferPair getFlattenFieldTransferPair(FieldReference outputName) {
 final TypedFieldId fieldId = incoming.getSchema().getFieldId(config.getColumn());
 final Class<? extends ValueVector> vectorClass = TypeHelper.getValueVectorClass(incoming.getSchema().getColumn(fieldId.getFieldIds()[0]));
 final ValueVector flattenField = incoming.getValueAccessorById(vectorClass, fieldId.getFieldIds()).getValueVector();
 final ValueVector vvIn = RepeatedValueVector.class.cast(flattenField).getDataVector();
 return vvIn.getTransferPair(outputName.getAsNamePart().getName(), context.getAllocator());
}
origin: dremio/dremio-oss

private void loadNextBatch(boolean first) throws IOException{
 Preconditions.checkArgument(batchIndex + 1 < batchIndexMax, "You tried to go beyond end of available batches to read.");
 container.zeroVectors();
 /* uncompress the data when de-serializing the spilled data into ArrowBufs */
 final VectorAccessibleSerializable serializer = new VectorAccessibleSerializable(allocator, compressSpilledBatch, compressSpilledBatchAllocator);
 serializer.readFromStream(inputStream);
 final VectorContainer incoming = serializer.get();
 Iterator<VectorWrapper<?>> wrapperIterator = incoming.iterator();
 if(first){
  for(VectorWrapper<?> wrapper : incoming){
   final ValueVector sourceVector = wrapper.getValueVector();
   TransferPair pair = sourceVector.getTransferPair(allocator);
   final ValueVector targetVector = pair.getTo();
   pair.transfer();
   container.add(targetVector);
  }
  container.buildSchema();
 }else{
  for (VectorWrapper<?> w : container) {
   final ValueVector sourceVector = wrapperIterator.next().getValueVector();
   final TransferPair pair = sourceVector.makeTransferPair(w.getValueVector());
   pair.transfer();
  }
 }
 recordIndexMax = incoming.getRecordCount();
 batchIndex++;
 recordIndex = -1;
}
origin: org.apache.arrow/arrow-vector

private FieldWriter promoteToUnion() {
 String name = vector.getField().getName();
 TransferPair tp = vector.getTransferPair(vector.getMinorType().name().toLowerCase(), vector.getAllocator());
 tp.transfer();
 if (parentContainer != null) {
  // TODO allow dictionaries in complex types
  unionVector = parentContainer.addOrGetUnion(name);
  unionVector.allocateNew();
 } else if (listVector != null) {
  unionVector = listVector.promoteToUnion();
 }
 unionVector.addVector((FieldVector) tp.getTo());
 writer = new UnionWriter(unionVector, nullableStructWriterFactory);
 writer.setPosition(idx());
 for (int i = 0; i <= idx(); i++) {
  unionVector.setType(i, vector.getMinorType());
 }
 vector = null;
 state = State.UNION;
 return writer;
}
origin: dremio/dremio-oss

TransferPair pair = w.getValueVector().getTransferPair(context.getAllocator());
atBatInput.add(pair.getTo());
transfers.add(pair);
org.apache.arrow.vectorValueVectorgetTransferPair

Javadoc

To transfer quota responsibility.

Popular methods of ValueVector

  • getField
    Get information about how this field is materialized.
  • getObject
    Get friendly type object from the vector.
  • isNull
    Check whether an element in the vector is null.
  • setValueCount
    Set number of values in the vector.
  • allocateNew
    Allocate new buffers. ValueVector implements logic to determine how much to allocate.
  • clear
    Release any owned ArrowBuf and reset the ValueVector to the initial state. If the vector has any chi
  • close
    Alternative to clear(). Allows use as an AutoCloseable in try-with-resources.
  • getValueCount
    Gets the number of values.
  • allocateNewSafe
    Allocates new buffers. ValueVector implements logic to determine how much to allocate.
  • getBufferSize
    Get the number of bytes used by this vector.
  • getBufferSizeFor
    Returns the number of bytes that is used by this vector if it holds the given number of values. The
  • getBuffers
    Return the underlying buffers associated with this vector. Note that this doesn't impact the referen
  • getBufferSizeFor,
  • getBuffers,
  • getMinorType,
  • getNullCount,
  • getReader,
  • setInitialCapacity,
  • getAllocator,
  • getDataBuffer,
  • getValidityBuffer

Popular in Java

  • Parsing JSON documents to java classes using gson
  • getSystemService (Context)
  • scheduleAtFixedRate (Timer)
  • requestLocationUpdates (LocationManager)
  • Container (java.awt)
    A generic Abstract Window Toolkit(AWT) container object is a component that can contain other AWT co
  • IOException (java.io)
    Signals a general, I/O-related error. Error details may be specified when calling the constructor, a
  • Time (java.sql)
    Java representation of an SQL TIME value. Provides utilities to format and parse the time's represen
  • Collection (java.util)
    Collection is the root of the collection hierarchy. It defines operations on data collections and t
  • Collections (java.util)
    This class consists exclusively of static methods that operate on or return collections. It contains
  • Pattern (java.util.regex)
    Patterns are compiled regular expressions. In many cases, convenience methods such as String#matches
  • Top 15 Vim Plugins
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimAtomGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyStudentsTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now