@Override public FieldVector getNewVector( String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new IntVector(name, fieldType, allocator); }
public TransferImpl(String ref, BufferAllocator allocator) { to = new IntVector(ref, field.getFieldType(), allocator); }
/** * * @param allocator * @param name * @return */ public static IntVector intVectorOf(BufferAllocator allocator,String name,int length) { IntVector float8Vector = new IntVector(name,FieldType.nullable(new ArrowType.Int(32,true)),allocator); float8Vector.allocateNew(length); float8Vector.setValueCount(length); return float8Vector; }
@SuppressWarnings({ "unchecked", "rawtypes" }) private VectorAccessible getBatch(MinorType type){ final IntVector vector = new IntVector("result", RootAllocatorFactory.newRoot(DEFAULT_SABOT_CONFIG)); VectorWrapper<IntVector> wrapper = Mockito.mock(VectorWrapper.class); when(wrapper.getValueVector()).thenReturn(vector); final TypedFieldId tfid = new TypedFieldId(CompleteType.fromMinorType(type), false, 0); VectorAccessible batch = Mockito.mock(VectorAccessible.class); when(batch.getValueVectorId(any(SchemaPath.class))).thenReturn(tfid); when(batch.getValueAccessorById(any(Class.class), any(int[].class))).thenReturn((VectorWrapper) wrapper); when(batch.getSchema()).thenReturn(BatchSchema.newBuilder().addField(MajorTypeHelper.getFieldForNameAndMajorType("result", Types.optional(type))).setSelectionVectorMode(SelectionVectorMode.NONE).build()); return batch; }
/** * * @param allocator * @param name * @param data * @return */ public static IntVector vectorFor(BufferAllocator allocator,String name,int[] data) { IntVector float8Vector = new IntVector(name,FieldType.nullable(new ArrowType.Int(32,true)),allocator); float8Vector.allocateNew(data.length); for(int i = 0; i < data.length; i++) { float8Vector.setSafe(i,data[i]); } float8Vector.setValueCount(data.length); return float8Vector; }
private AccumulatorSet createAccumulator(IntVector in1, final BufferAllocator allocator) { /* SUM Accumulator */ BigIntVector in1SumOutputVector = new BigIntVector("int-sum", allocator); final SumAccumulators.IntSumAccumulator in1SumAccum = new SumAccumulators.IntSumAccumulator(in1, in1SumOutputVector, in1SumOutputVector, MAX_VALUES_PER_BATCH, allocator); /* Min Accumulator */ IntVector in1MaxOutputVector = new IntVector("int-max", allocator); final MaxAccumulators.IntMaxAccumulator in1MaxAccum = new MaxAccumulators.IntMaxAccumulator(in1, in1MaxOutputVector, in1MaxOutputVector, MAX_VALUES_PER_BATCH, allocator); return new AccumulatorSet(4*1024, 64*1024, allocator, in1SumAccum, in1MaxAccum); }
private static Pair<IntVector, ResultVerifier> testIntVector(final int startIndexInCurrentOutput, final int startIndexInJob) { IntVector colIntV = new IntVector("colInt", allocator); colIntV.allocateNew(5); colIntV.set(0, 20); colIntV.set(1, 50); colIntV.set(2, -2000); colIntV.set(3, 327345); colIntV.setNull(4); ResultVerifier verifier = new ResultVerifier() { @Override public void verify(DataPOJO output) { int index = startIndexInCurrentOutput; assertEquals(20, ((Integer)output.extractValue("colInt", 0)).intValue()); assertEquals(50, ((Integer)output.extractValue("colInt", 1)).intValue()); assertEquals(-2000, ((Integer)output.extractValue("colInt", 2)).intValue()); assertEquals(327345, ((Integer)output.extractValue("colInt", 3)).intValue()); assertNull(output.extractValue("colInt", 4)); } }; return Pair.of(colIntV, verifier); }
IntVector in1MaxOutputVector = new IntVector("int-max", allocator); final MaxAccumulators.IntMaxAccumulator in1MaxAccum = new MaxAccumulators.IntMaxAccumulator(in1, in1MaxOutputVector, in1MaxOutputVector, MAX_VALUES_PER_BATCH, IntVector in1MinOutputVector = new IntVector("int-min", allocator); final MinAccumulators.IntMinAccumulator in1MinAccum = new MinAccumulators.IntMinAccumulator(in1, in1MinOutputVector, in1MinOutputVector, MAX_VALUES_PER_BATCH,
private IntVector intVector(String name) { IntVector vec = new IntVector(name, allocator); vec.allocateNew(5); vec.set(0, 20); vec.set(1, 50); vec.set(2, -2000); vec.set(3, 327345); vec.setNull(4); vec.setValueCount(COUNT); return vec; }
private void testDualIntVectors(int count, Integer[] vectA, Integer[] vectB){ try( IntVector col1 = new IntVector("col1", allocator); IntVector col2 = new IntVector("col2", allocator);){ PivotDef pivot = PivotBuilder.getBlockDefinition(new FieldVectorPair(col1, col1), new FieldVectorPair(col2, col2) ); populate(col1, vectA); populate(col2, vectB); assertEquals(12, pivot.getBlockWidth()); try( FixedBlockVector fixed = new FixedBlockVector(allocator, pivot.getBlockWidth()); VariableBlockVector variable = new VariableBlockVector(allocator, pivot.getVariableCount()); ){ Pivots.pivot(pivot, count, fixed, variable); validateIntValues(pivot.getBlockWidth(), pivot.getFixedPivots().get(0), fixed, vectA); validateIntValues(pivot.getBlockWidth(), pivot.getFixedPivots().get(1), fixed, vectB); } } }
@Override public void setup(VectorAccessible incoming) throws Exception { state.is(State.NEEDS_SETUP); Preconditions.checkState(incoming.getSchema().getSelectionVectorMode() == BatchSchema.SelectionVectorMode.NONE, "Vectorized Partition Sender doesn't support SV " + incoming.getSchema().getSelectionVectorMode()); checkSchema(incoming.getSchema()); // how many records we can keep in memory before we are forced to flush the outgoing batch numRecordsBeforeFlush = calculateBucketSize(incoming); stats.setLongStat(Metric.BUCKET_SIZE, numRecordsBeforeFlush); // final BufferAllocator allocator = context.getAllocator(); // we need to synchronize this to ensure no receiver termination message is lost synchronized (batchCreationLock) { initBatchesAndLookup(incoming); // some receivers may have finished already, make sure to terminate corresponding outgoing batches for (int index = 0; index < terminations.size(); index++) { final int p = terminations.buffer[index]; batches[p].terminate(); batches[p + numReceivers].terminate(); } terminations.clear(); } copiers = MultiDestCopier.getCopiers(VectorContainer.getFieldVectors(incoming), batches, copyWatches); copyIndices = new IntVector("copy-compound-indices", allocator); copyIndices.allocateNew(numRecordsBeforeFlush); initHashVector(incoming); state = State.CAN_CONSUME; }
@Test public void testIntPivot(){ final Integer[] vectA = {15, null, Integer.MAX_VALUE - 45, null, Integer.MIN_VALUE + 17}; final Integer[] vectB = {null, 45, null, 65, 77}; try( IntVector col1 = new IntVector("col1", allocator); IntVector col2 = new IntVector("col2", allocator);){ PivotDef pivot = PivotBuilder.getBlockDefinition( new FieldVectorPair(col1, col1), new FieldVectorPair(col2, col2) ); populate(col1, vectA); populate(col2, vectB); assertEquals(12, pivot.getBlockWidth()); try( FixedBlockVector fixed = new FixedBlockVector(allocator, pivot.getBlockWidth()); VariableBlockVector variable = new VariableBlockVector(allocator, pivot.getVariableCount()); ){ Pivots.pivot(pivot, 5, fixed, variable); validateIntValues(pivot.getBlockWidth(), pivot.getFixedPivots().get(0), fixed, vectA); validateIntValues(pivot.getBlockWidth(), pivot.getFixedPivots().get(1), fixed, vectB); } } }
@Test public void intRoundtrip() throws Exception { final int count = 1024; try ( IntVector in = new IntVector("in", allocator); IntVector out = new IntVector("out", allocator); ) { in.allocateNew(count); for (int i = 0; i < count; i++) { if (i % 5 == 0) { in.setSafe(i, i); } } in.setValueCount(count); final PivotDef pivot = PivotBuilder.getBlockDefinition(new FieldVectorPair(in, out)); try ( final FixedBlockVector fbv = new FixedBlockVector(allocator, pivot.getBlockWidth()); final VariableBlockVector vbv = new VariableBlockVector(allocator, pivot.getVariableCount()); ) { fbv.ensureAvailableBlocks(count); Pivots.pivot4Bytes(pivot.getFixedPivots().get(0), fbv, count); ValueVector[] ins = new ValueVector[]{in}; ValueVector[] outs = new ValueVector[]{out}; unpivotHelper(pivot, fbv, vbv, ins, outs, 0, count); unpivotHelper(pivot, fbv, vbv, ins, outs, 0, 100); unpivotHelper(pivot, fbv, vbv, ins, outs, 100, 924); } } }
@Test public void fixedVariableWithoutNull(){ try(IntVector col1 = new IntVector("col1", allocator); BigIntVector col2 = new BigIntVector("col2", allocator); VarCharVector col3 = new VarCharVector("col3", allocator);){
@Test public void fixedVariable(){ try(IntVector col1 = new IntVector("col1", allocator); BigIntVector col2 = new BigIntVector("col2", allocator); VarCharVector col3 = new VarCharVector("col3", allocator);){
@Test public void fixedOnlyWithoutNull(){ try(IntVector col1 = new IntVector("col1", allocator); BigIntVector col2 = new BigIntVector("col2", allocator); DecimalVector col3 = new DecimalVector("col3", allocator, 30, 0);
@Test public void fixedOnly(){ try(IntVector col1 = new IntVector("col1", allocator); BigIntVector col2 = new BigIntVector("col2", allocator); DecimalVector col3 = new DecimalVector("col3", allocator, 30, 0);
VarCharVector col2 = new VarCharVector("col2", allocator); c.add(col2); IntVector col3 = new IntVector("col3", allocator); c.add(col3); c.allocateNew();