Tabnine Logo
ValueVector
Code IndexAdd Tabnine to your IDE (free)

How to use
ValueVector
in
org.apache.arrow.vector

Best Java code snippets using org.apache.arrow.vector.ValueVector (Showing top 20 results out of 315)

origin: dremio/dremio-oss

@Override
public Field getField() {
 return vector.getField();
}
origin: dremio/dremio-oss

@Override
public Object getObject(int index) throws InvalidAccessException {
 return v.getObject(index);
}
origin: org.apache.arrow/arrow-vector

private boolean nullFilled(ValueVector vector) {
 for (int r = 0; r < vector.getValueCount(); r++) {
  if (!vector.isNull(r)) {
   return false;
  }
 }
 return true;
}
origin: org.apache.arrow/arrow-vector

/**
 * Decodes a dictionary encoded array using the provided dictionary.
 *
 * @param indices    dictionary encoded values, must be int type
 * @param dictionary dictionary used to decode the values
 * @return vector with values restored from dictionary
 */
public static ValueVector decode(ValueVector indices, Dictionary dictionary) {
 int count = indices.getValueCount();
 ValueVector dictionaryVector = dictionary.getVector();
 int dictionaryCount = dictionaryVector.getValueCount();
 // copy the dictionary values into the decoded vector
 TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator());
 transfer.getTo().allocateNewSafe();
 for (int i = 0; i < count; i++) {
  Object index = indices.getObject(i);
  if (index != null) {
   int indexAsInt = ((Number) index).intValue();
   if (indexAsInt > dictionaryCount) {
    throw new IllegalArgumentException("Provided dictionary does not contain value for index " + indexAsInt);
   }
   transfer.copyValueSafe(indexAsInt, i);
  }
 }
 // TODO do we need to worry about the field?
 ValueVector decoded = transfer.getTo();
 decoded.setValueCount(count);
 return decoded;
}
origin: org.apache.arrow/arrow-vector

@Override
public Object getObject(int index) {
 Map<String, Object> vv = new JsonStringHashMap<>();
 for (String child : getChildFieldNames()) {
  ValueVector v = getChild(child);
  if (v != null && index < v.getValueCount()) {
   Object value = v.getObject(index);
   if (value != null) {
    vv.put(child, value);
   }
  }
 }
 return vv;
}
origin: dremio/dremio-oss

public void setValueCount(int valueCount){
 vector.setValueCount(valueCount);
}
origin: dremio/dremio-oss

@Override
public void dataArrived(QueryDataBatch result, ConnectionThrottle throttle) {
 QueryData queryHeader = result.getHeader();
 int rows = queryHeader.getRowCount();
 try {
  if ( result.hasData() ) {
   ArrowBuf data = result.getData();
   loader.load(queryHeader.getDef(), data);
   for (int i = 0; i < rows; i++) {
     Map<String,String> record = Maps.newHashMap();
    for (VectorWrapper<?> vw : loader) {
     final String field = vw.getValueVector().getField().getName();
     final ValueVector vv = vw.getValueVector();
     final Object value = i < vv.getValueCount() ? vv.getObject(i) : null;
     final String display = value == null ? null : value.toString();
     record.put(field, display);
    }
    records.add(record);
   }
   loader.clear();
  }
  result.release();
 } catch (SchemaChangeException e) {
  fail(e.getMessage());
 }
}
origin: dremio/dremio-oss

@Override
public void allocate() {
 vector.allocateNew();
}
origin: dremio/dremio-oss

@Override
public void clear() {
 if (!releasable) {
  return;
 }
 for (T x : vectors) {
  x.clear();
 }
}
origin: org.apache.arrow/arrow-vector

@Override
public TimeStampNanoTZWriter timeStampNanoTZ(String name, String timezone) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  TimeStampNanoTZVector v = container.addOrGet(name, 
    FieldType.nullable(
     new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, timezone)
    ),
    TimeStampNanoTZVector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.TIMESTAMPNANOTZ);
  }
 }
 return writer;
}
origin: dremio/dremio-oss

@Before
public void setUp() throws Exception {
 valueVector = mock(ValueVector.class);
 when(valueVector.getObject(anyInt())).thenAnswer(new Answer<Object>() {
 when(valueVector.isNull(anyInt())).thenAnswer(new Answer<Object>() {
origin: dremio/dremio-oss

@Override
public int outputData() {
 if(checkForStraightCopy && incoming.getRecordCount() == randomVector.getValueCount()){
  for(TransferPair tp : transferPairs){
   tp.transfer();
  }
  output.setRecordCount(incoming.getRecordCount());
  state = State.CAN_CONSUME;
  return incoming.getRecordCount();
 }
 int recordCount = incoming.getRecordCount() - this.copyOffset;
 int copiedRecords = copier.copyRecords(copyOffset, recordCount);
 if(copiedRecords < recordCount){
  copyOffset = copyOffset + copiedRecords;
 }else{
  copyOffset = 0;
  state = State.CAN_CONSUME;
 }
 if(copiedRecords > 0){
  for(VectorWrapper<?> v : output){
   v.getValueVector().setValueCount(copiedRecords);
  }
 }
 output.setRecordCount(copiedRecords);
 return copiedRecords;
}
origin: dremio/dremio-oss

public void getFieldById(int fieldId, ComplexHolder holder) {
 final ValueVector vv = vectors[fieldId];
 if (vv instanceof BaseValueVector) {
  holder.isSet =vv.isNull(currentIndex) ? 1 : 0;
 } else {
  holder.isSet = vv.isNull(currentIndex) ? 1 : 0;
 }
 holder.reader = vv.getReader();
 holder.reader.setPosition(currentIndex);
}
origin: dremio/dremio-oss

@SuppressWarnings("unchecked")
@Override
public VectorWrapper<T> cloneAndTransfer(BufferAllocator allocator, CallBack callback) {
 try{
 TransferPair tp = vector.getTransferPair(vector.getField().getName(), allocator, callback);
 tp.transfer();
 return new SimpleVectorWrapper<T>((T) tp.getTo());
 }catch(RuntimeException ex){
  throw ex;
 }
}
origin: org.apache.spark/spark-sql_2.11

final void close() {
 vector.close();
}
origin: dremio/dremio-oss

 /**
  * Returns an instance sitting at the given index if exists, null otherwise.
  *
  * @see com.dremio.exec.vector.accessor.SqlAccessor#getObject(int)
  */
 @Override
 public Object getObject(int rowOffset) throws InvalidAccessException {
  // In case some vectors have fewer values than others, and callee invokes
  // this method with index >= getValueCount(), this should still yield null.
  if (rowOffset < vector.getValueCount()) {
   return delegate.getObject(rowOffset);
  }
  return null;
 }
}
origin: org.apache.spark/spark-sql_2.11

boolean isNullAt(int rowId) {
 return vector.isNull(rowId);
}
origin: org.apache.arrow/arrow-vector

@Override
public ArrowBuf[] getBuffers(boolean clear) {
 final List<ArrowBuf> buffers = new ArrayList<>();
 for (final ValueVector vector : vectors.values()) {
  for (final ArrowBuf buf : vector.getBuffers(false)) {
   buffers.add(buf);
   if (clear) {
    buf.retain(1);
   }
  }
  if (clear) {
   vector.clear();
  }
 }
 return buffers.toArray(new ArrowBuf[buffers.size()]);
}
origin: dremio/dremio-oss

public boolean allocateNewSafe() {
 for (VectorWrapper<?> w : wrappers) {
  if (!w.getValueVector().allocateNewSafe()) {
   return false;
  }
 }
 return true;
}
origin: dremio/dremio-oss

@Override
public Object next() {
 if (currVec == null || indexInCurrentVector == currVec.getValueCount()) {
  currVec = hyperVector.getValueVectors()[indexInVectorList];
  indexInVectorList++;
  indexInCurrentVector = 0;
 }
 Object obj = currVec.getObject(indexInCurrentVector);
 indexInCurrentVector++;
 totalValuesRead++;
 return obj;
}
org.apache.arrow.vectorValueVector

Javadoc

An abstraction that is used to store a sequence of values in an individual column.

A ValueVector stores underlying data in-memory in a columnar fashion that is compact and efficient. The column whose data is stored, is referred by #getField().

It is important that vector is allocated before attempting to read or write.

There are a few "rules" around vectors:

  • values need to be written in order (e.g. index 0, 1, 2, 5)
  • null vectors start with all values as null before writing anything
  • for variable width types, the offset vector should be all zeros before writing
  • you must call setValueCount before a vector can be read
  • you should never write to a vector once it has been read.

Please note that the current implementation doesn't enforce those rules, hence we may find few places that deviate from these rules (e.g. offset vectors in Variable Length and Repeated vector)

This interface "should" strive to guarantee this order of operation:

allocate > mutate > setvaluecount > access > clear (or allocate to start the process over).

Most used methods

  • getField
    Get information about how this field is materialized.
  • getObject
    Get friendly type object from the vector.
  • isNull
    Check whether an element in the vector is null.
  • setValueCount
    Set number of values in the vector.
  • allocateNew
    Allocate new buffers. ValueVector implements logic to determine how much to allocate.
  • clear
    Release any owned ArrowBuf and reset the ValueVector to the initial state. If the vector has any chi
  • close
    Alternative to clear(). Allows use as an AutoCloseable in try-with-resources.
  • getValueCount
    Gets the number of values.
  • allocateNewSafe
    Allocates new buffers. ValueVector implements logic to determine how much to allocate.
  • getBufferSize
    Get the number of bytes used by this vector.
  • getBufferSizeFor
    Returns the number of bytes that is used by this vector if it holds the given number of values. The
  • getBuffers
    Return the underlying buffers associated with this vector. Note that this doesn't impact the referen
  • getBufferSizeFor,
  • getBuffers,
  • getMinorType,
  • getNullCount,
  • getReader,
  • getTransferPair,
  • setInitialCapacity,
  • getAllocator,
  • getDataBuffer,
  • getValidityBuffer

Popular in Java

  • Making http post requests using okhttp
  • findViewById (Activity)
  • setRequestProperty (URLConnection)
  • scheduleAtFixedRate (ScheduledExecutorService)
  • ObjectMapper (com.fasterxml.jackson.databind)
    ObjectMapper provides functionality for reading and writing JSON, either to and from basic POJOs (Pl
  • Thread (java.lang)
    A thread is a thread of execution in a program. The Java Virtual Machine allows an application to ha
  • SocketException (java.net)
    This SocketException may be thrown during socket creation or setting options, and is the superclass
  • Date (java.sql)
    A class which can consume and produce dates in SQL Date format. Dates are represented in SQL as yyyy
  • NoSuchElementException (java.util)
    Thrown when trying to retrieve an element past the end of an Enumeration or Iterator.
  • Response (javax.ws.rs.core)
    Defines the contract between a returned instance and the runtime when an application needs to provid
  • CodeWhisperer alternatives
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now