How to use

setInitialCapacity

method

in

org.apache.arrow.vector.ValueVector

@Override
public void setInitialCapacity(int numRecords) {
 for (final ValueVector v : (Iterable<ValueVector>) this) {
  v.setInitialCapacity(numRecords);
 }
}

@Override
public void setInitialCapacity(int valueCount, double density) {
 for (final ValueVector vector : (Iterable<ValueVector>) this) {
  if (vector instanceof DensityAwareVector) {
   ((DensityAwareVector)vector).setInitialCapacity(valueCount, density);
  } else {
   vector.setInitialCapacity(valueCount);
  }
 }
}

public void setInitialCapacity(int initialCapacity) {
 for (VectorWrapper<?> w : wrappers) {
  final ValueVector vv = w.getValueVector();
  vv.setInitialCapacity(initialCapacity);
 }
}

OutgoingBatch(int batchIdx, int nextBatchIdx, int maxRecords, final VectorAccessible incoming,
       BufferAllocator allocator, AccountingExecTunnel tunnel, HashPartitionSender config,
       OperatorContext context, int oppositeMinorFragmentId, OperatorStats stats) {
 Preconditions.checkArgument(maxRecords <= Character.MAX_VALUE, "maxRecords cannot exceed " + Character.MAX_VALUE);
 this.batchIdx = batchIdx;
 this.nextBatchIdx = nextBatchIdx;
 this.maxRecords = maxRecords;
 this.tunnel = tunnel;
 this.config = config;
 this.context = context;
 this.oppositeMinorFragmentId = oppositeMinorFragmentId;
 this.stats = stats;
 for (VectorWrapper<?> v : incoming) {
  ValueVector outgoingVector = TypeHelper.getNewVector(v.getField(), allocator);
  outgoingVector.setInitialCapacity(maxRecords);
  add(outgoingVector);
  if (outgoingVector instanceof VarBinaryVector) {
   varbins.add(((VarBinaryVector) outgoingVector));
  } else if (outgoingVector instanceof VarCharVector) {
   varchars.add(((VarCharVector) outgoingVector));
  }
 }
}

private void allocateVectors(int targetRecordCount) {
 boolean memoryAllocated = false;
 double density = lastSuccessfulDensity;
 while (!memoryAllocated) {
  try {
   for (VectorWrapper<?> w : outgoing) {
    final ValueVector v = w.getValueVector();
    if (v instanceof DensityAwareVector) {
     ((DensityAwareVector) v).setInitialCapacity(targetRecordCount, density);
    } else {
     v.setInitialCapacity(targetRecordCount);
    }
    v.allocateNew();
   }
   memoryAllocated = true;
   lastSuccessfulDensity = density;
  } catch (OutOfMemoryException ex) {
   // halve the density and try again
   density = density / 2;
   if (density < 0.01) {
    logger.debug("PriorityQueueCopierTemplate ran out of memory to allocate outgoing batch. " +
      "Records: {}, density: {}", targetRecordCount, density);
    throw ex;
   }
   // try allocating again with lower density
   logger.debug("PriorityQueueCopierTemplate: Ran out of memory. Retrying allocation with lower density.");
  }
 }
}

@Override
public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException {
 int estimatedRecordCount;
 if ((reader != null) && (reader.getInput() != null) && (vectorMap.size() > 0)) {
  final OptionManager options = context.getOptions();
  final int listSizeEstimate = (int) options.getOption(ExecConstants.BATCH_LIST_SIZE_ESTIMATE);
  final int varFieldSizeEstimate = (int) options.getOption(ExecConstants.BATCH_VARIABLE_FIELD_SIZE_ESTIMATE);
  final int estimatedRecordSize = BatchSchema.estimateRecordSize(vectorMap, listSizeEstimate, varFieldSizeEstimate);
  if (estimatedRecordSize > 0) {
   estimatedRecordCount = (int) Math.min(reader.getInput().length / estimatedRecordSize, numRowsPerBatch);
  } else {
   estimatedRecordCount = (int) numRowsPerBatch;
  }
 } else {
  estimatedRecordCount = (int) numRowsPerBatch;
 }
 for (final ValueVector v : vectorMap.values()) {
  v.setInitialCapacity(estimatedRecordCount);
  v.allocateNew();
 }
}

/**
 * Initialize the OutgoingBatch based on the current schema in incoming RecordBatch
 */
public void initializeBatch() {
 for (VectorWrapper<?> v : incoming) {
  // create new vector
  ValueVector outgoingVector = TypeHelper.getNewVector(v.getField(), allocator);
  outgoingVector.setInitialCapacity(maxRecordCount);
  vectorContainer.add(outgoingVector);
 }
 vectorContainer.allocateNew();
 doSetup(incoming, vectorContainer);
}

@Override
public DateDayWriter dateDay(String name) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  DateDayVector v = container.addOrGet(name, 
    FieldType.nullable(
     MinorType.DATEDAY.getType()
    ),
    DateDayVector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.DATEDAY);
  }
 }
 return writer;
}

@Override
public TimeStampNanoTZWriter timeStampNanoTZ(String name, String timezone) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  TimeStampNanoTZVector v = container.addOrGet(name, 
    FieldType.nullable(
     new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, timezone)
    ),
    TimeStampNanoTZVector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.TIMESTAMPNANOTZ);
  }
 }
 return writer;
}

@Override
public DecimalWriter decimal(String name, int scale, int precision) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  DecimalVector v = container.addOrGet(name, 
    FieldType.nullable(
     new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)
    ),
    DecimalVector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.DECIMAL);
  }
 }
 return writer;
}

@Override
public UInt1Writer uInt1(String name) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  UInt1Vector v = container.addOrGet(name, 
    FieldType.nullable(
     MinorType.UINT1.getType()
    ),
    UInt1Vector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.UINT1);
  }
 }
 return writer;
}

@Override
public UInt4Writer uInt4(String name) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  UInt4Vector v = container.addOrGet(name, 
    FieldType.nullable(
     MinorType.UINT4.getType()
    ),
    UInt4Vector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.UINT4);
  }
 }
 return writer;
}

@Override
public DateDayWriter dateDay(String name) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  DateDayVector v = container.addOrGet(name, 
    FieldType.nullable(
     MinorType.DATEDAY.getType()
    ),
    DateDayVector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.DATEDAY);
  }
 }
 return writer;
}

@Override
public IntervalYearWriter intervalYear(String name) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  IntervalYearVector v = container.addOrGet(name, 
    FieldType.nullable(
     MinorType.INTERVALYEAR.getType()
    ),
    IntervalYearVector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.INTERVALYEAR);
  }
 }
 return writer;
}

@Override
public UInt8Writer uInt8(String name) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  UInt8Vector v = container.addOrGet(name, 
    FieldType.nullable(
     MinorType.UINT8.getType()
    ),
    UInt8Vector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.UINT8);
  }
 }
 return writer;
}

@Override
public TimeStampNanoWriter timeStampNano(String name) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  TimeStampNanoVector v = container.addOrGet(name, 
    FieldType.nullable(
     MinorType.TIMESTAMPNANO.getType()
    ),
    TimeStampNanoVector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.TIMESTAMPNANO);
  }
 }
 return writer;
}

@Override
public TimeStampMicroTZWriter timeStampMicroTZ(String name, String timezone) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  TimeStampMicroTZVector v = container.addOrGet(name, 
    FieldType.nullable(
     new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, timezone)
    ),
    TimeStampMicroTZVector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.TIMESTAMPMICROTZ);
  }
 }
 return writer;
}

@Override
public Float8Writer float8(String name) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  Float8Vector v = container.addOrGet(name, 
    FieldType.nullable(
     MinorType.FLOAT8.getType()
    ),
    Float8Vector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.FLOAT8);
  }
 }
 return writer;
}

@Override
public TimeMicroWriter timeMicro(String name) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  TimeMicroVector v = container.addOrGet(name, 
    FieldType.nullable(
     MinorType.TIMEMICRO.getType()
    ),
    TimeMicroVector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.TIMEMICRO);
  }
 }
 return writer;
}

@Override
public DecimalWriter decimal(String name, int scale, int precision) {
 FieldWriter writer = fields.get(handleCase(name));
 if(writer == null) {
  ValueVector vector;
  ValueVector currentVector = container.getChild(name);
  DecimalVector v = container.addOrGet(name, 
    FieldType.nullable(
     new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale)
    ),
    DecimalVector.class);
  writer = new PromotableWriter(v, container, getNullableStructWriterFactory());
  vector = v;
  if (currentVector == null || currentVector != vector) {
   if(this.initialCapacity > 0) {
    vector.setInitialCapacity(this.initialCapacity);
   }
   vector.allocateNewSafe();
  } 
  writer.setPosition(idx());
  fields.put(handleCase(name), writer);
 } else {
  if (writer instanceof PromotableWriter) {
   // ensure writers are initialized
   ((PromotableWriter)writer).getWriter(MinorType.DECIMAL);
  }
 }
 return writer;
}