@Override public void allocate() { vector.allocateNew(); }
public void allocate(){ vector.allocateNew(); }
static Integer[] populate4ByteValues(IntVector vector, int size){ vector.allocateNew(); Integer values[] = new Integer[size]; for(int i =0; i < size; i++){ if(RAND.nextBoolean()){ values[i] = RAND.nextInt(); vector.setSafe(i, values[i]); } } vector.setValueCount(size); return values; }
static Integer[] populate4ByteValuesWithoutNull(IntVector vector, int size){ vector.allocateNew(); Integer values[] = new Integer[size]; for(int i =0; i < size; i++){ values[i] = RAND.nextInt(); vector.setSafe(i, values[i]); } vector.setValueCount(size); return values; }
private void populateInt(IntVector vector, Integer[] data) { vector.allocateNew(); Random r = new Random(); for(int i =0; i < data.length; i++){ Integer val = data[i]; if(val != null){ vector.setSafe(i, val); } else { vector.setSafe(i, 0, r.nextInt()); } } vector.setValueCount(data.length); }
private void populateInt(IntVector vector, Integer[] data) { vector.allocateNew(); Random r = new Random(); for(int i =0; i < data.length; i++){ Integer val = data[i]; if(val != null){ vector.setSafe(i, val); } else { vector.setSafe(i, 0, r.nextInt()); } } vector.setValueCount(data.length); }
/** * * @param allocator * @param name * @return */ public static IntVector intVectorOf(BufferAllocator allocator,String name,int length) { IntVector float8Vector = new IntVector(name,FieldType.nullable(new ArrowType.Int(32,true)),allocator); float8Vector.allocateNew(length); float8Vector.setValueCount(length); return float8Vector; }
private void populateInt(IntVector vector, Integer[] data) { vector.allocateNew(); Random r = new Random(); for(int i =0; i < data.length; i++){ Integer val = data[i]; if(val != null){ vector.setSafe(i, val); } else { vector.setSafe(i, 0, r.nextInt()); } } vector.setValueCount(data.length); }
static void populate(IntVector vector, Integer[] values){ vector.allocateNew(); Random r = new Random(); for(int i =0; i < values.length; i++){ Integer val = values[i]; if(val != null){ vector.setSafe(i, val); } else { // add noise so we make sure not to read/see noise. vector.setSafe(i, 0, r.nextInt()); } } vector.setValueCount(values.length); }
/** * * @param allocator * @param name * @param data * @return */ public static IntVector vectorFor(BufferAllocator allocator,String name,int[] data) { IntVector float8Vector = new IntVector(name,FieldType.nullable(new ArrowType.Int(32,true)),allocator); float8Vector.allocateNew(data.length); for(int i = 0; i < data.length; i++) { float8Vector.setSafe(i,data[i]); } float8Vector.setValueCount(data.length); return float8Vector; }
private static VectorContainer buildIntegerGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) { final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Int(32, true), null); final VectorContainer input = new VectorContainer(bufferAllocator); final IntVector intVector = input.addOrGet(field); intVector.allocateNew(); final SortedSet<Integer> values = Sets.newTreeSet(); for (Dictionary dictionary : dictionaries) { for (int i = 0; i <= dictionary.getMaxId(); ++i) { values.add(dictionary.decodeToInt(i)); } } if (existingDict != null) { final IntVector existingDictValues = existingDict.getValueAccessorById(IntVector.class, 0).getValueVector(); for (int i = 0; i < existingDict.getRecordCount(); ++i) { values.add(existingDictValues.get(i)); } } final Iterator<Integer> iter = values.iterator(); int recordCount = 0; while (iter.hasNext()) { intVector.setSafe(recordCount++, iter.next()); } intVector.setValueCount(recordCount); input.setRecordCount(recordCount); input.buildSchema(BatchSchema.SelectionVectorMode.NONE); return input; }
public void loadData(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int valuesLength = buffer.capacity(); vector.allocateNew(metadata.getValueCount()); /* set inner validity buffer */ setValidityBuffer(bitsField); /* load inner value buffer */ vector.valueBuffer.close(); loadDataBuffer(valuesField, buffer.slice(0, valuesLength)); }
private static Pair<IntVector, ResultVerifier> testIntVector(final int startIndexInCurrentOutput, final int startIndexInJob) { IntVector colIntV = new IntVector("colInt", allocator); colIntV.allocateNew(5); colIntV.set(0, 20); colIntV.set(1, 50); colIntV.set(2, -2000); colIntV.set(3, 327345); colIntV.setNull(4); ResultVerifier verifier = new ResultVerifier() { @Override public void verify(DataPOJO output) { int index = startIndexInCurrentOutput; assertEquals(20, ((Integer)output.extractValue("colInt", 0)).intValue()); assertEquals(50, ((Integer)output.extractValue("colInt", 1)).intValue()); assertEquals(-2000, ((Integer)output.extractValue("colInt", 2)).intValue()); assertEquals(327345, ((Integer)output.extractValue("colInt", 3)).intValue()); assertNull(output.extractValue("colInt", 4)); } }; return Pair.of(colIntV, verifier); }
private IntVector intVector(String name) { IntVector vec = new IntVector(name, allocator); vec.allocateNew(5); vec.set(0, 20); vec.set(1, 50); vec.set(2, -2000); vec.set(3, 327345); vec.setNull(4); vec.setValueCount(COUNT); return vec; }
@Override public void setup(VectorAccessible incoming) throws Exception { state.is(State.NEEDS_SETUP); Preconditions.checkState(incoming.getSchema().getSelectionVectorMode() == BatchSchema.SelectionVectorMode.NONE, "Vectorized Partition Sender doesn't support SV " + incoming.getSchema().getSelectionVectorMode()); checkSchema(incoming.getSchema()); // how many records we can keep in memory before we are forced to flush the outgoing batch numRecordsBeforeFlush = calculateBucketSize(incoming); stats.setLongStat(Metric.BUCKET_SIZE, numRecordsBeforeFlush); // final BufferAllocator allocator = context.getAllocator(); // we need to synchronize this to ensure no receiver termination message is lost synchronized (batchCreationLock) { initBatchesAndLookup(incoming); // some receivers may have finished already, make sure to terminate corresponding outgoing batches for (int index = 0; index < terminations.size(); index++) { final int p = terminations.buffer[index]; batches[p].terminate(); batches[p + numReceivers].terminate(); } terminations.clear(); } copiers = MultiDestCopier.getCopiers(VectorContainer.getFieldVectors(incoming), batches, copyWatches); copyIndices = new IntVector("copy-compound-indices", allocator); copyIndices.allocateNew(numRecordsBeforeFlush); initHashVector(incoming); state = State.CAN_CONSUME; }
final List<FieldVectorPair> probeFields = new ArrayList<>(); intInKey.allocateNew(count);
){ in.allocateNew(count);
IntVector inv = new IntVector("in", allocator); in[x] = inv; inv.allocateNew(count); IntVector outv = new IntVector("out", allocator); out[x] = outv;
@Test public void intRoundtrip() throws Exception { final int count = 1024; try ( IntVector in = new IntVector("in", allocator); IntVector out = new IntVector("out", allocator); ) { in.allocateNew(count); for (int i = 0; i < count; i++) { if (i % 5 == 0) { in.setSafe(i, i); } } in.setValueCount(count); final PivotDef pivot = PivotBuilder.getBlockDefinition(new FieldVectorPair(in, out)); try ( final FixedBlockVector fbv = new FixedBlockVector(allocator, pivot.getBlockWidth()); final VariableBlockVector vbv = new VariableBlockVector(allocator, pivot.getVariableCount()); ) { fbv.ensureAvailableBlocks(count); Pivots.pivot4Bytes(pivot.getFixedPivots().get(0), fbv, count); ValueVector[] ins = new ValueVector[]{in}; ValueVector[] outs = new ValueVector[]{out}; unpivotHelper(pivot, fbv, vbv, ins, outs, 0, count); unpivotHelper(pivot, fbv, vbv, ins, outs, 0, 100); unpivotHelper(pivot, fbv, vbv, ins, outs, 100, 924); } } }