/** * * @param allocator * @param name * @param data * @return */ public static VarCharVector vectorFor(BufferAllocator allocator,String name,String[] data) { VarCharVector float4Vector = new VarCharVector(name,allocator); float4Vector.allocateNew(); for(int i = 0; i < data.length; i++) { float4Vector.setSafe(i,data[i].getBytes()); } float4Vector.setValueCount(data.length); return float4Vector; }
@Override public void clear() { vector.clear(); }
public void populate(final int count){ for (int i = 0; i < count; i++) { if(value != null) { ((VarCharVector) vector).setSafe(i, value, 0, value.length); } } vector.setValueCount(count); }
/** * * @param allocator * @param name * @param length the length of the vector * @return */ public static VarCharVector stringVectorOf(BufferAllocator allocator,String name,int length) { VarCharVector float4Vector = new VarCharVector(name,allocator); float4Vector.allocateNew(); float4Vector.setValueCount(length); return float4Vector; }
@Test public void test() throws Exception { final VarCharVector varCharVector = new VarCharVector("field", allocator); varCharVector.allocateNew(10000, 1000); varCharVector.set(i, s.getBytes()); compareArray[i] = s; varCharVector.setValueCount(valueCount); final TransferPair tp = varCharVector.getTransferPair(allocator); final VarCharVector newVarCharVector = (VarCharVector) tp.getTo(); final int[][] startLengths = {{0, 201}, {201, 200}, {401, 99}}; final int length = startLength[1]; tp.splitAndTransfer(start, length); newVarCharVector.setValueCount(length); for (int i = 0; i < length; i++) { final boolean expectedSet = ((start + i) % 3) == 0; if (expectedSet) { final byte[] expectedValue = compareArray[start + i].getBytes(); assertFalse(newVarCharVector.isNull(i)); assertArrayEquals(expectedValue, newVarCharVector.get(i)); } else { assertTrue(newVarCharVector.isNull(i)); newVarCharVector.clear(); varCharVector.close(); allocator.close();
/** Helper method which creates a test varchar vector */ private static VarCharVector testVarCharVector() { VarCharVector colVarCharV = new VarCharVector("colVarChar", ALLOCATOR); colVarCharV.allocateNew(500, 5); for(int i=0; i<TEST_VARCHAR_VALUES.size(); i++) { if (TEST_VARCHAR_VALUES.get(i) == null) { colVarCharV.setNull(i); } else { colVarCharV.set(i, TEST_VARCHAR_VALUES.get(i).getBytes()); } } return colVarCharV; }
@Test public void testPrimitiveCharPadding() throws SerDeException { String[][] schema = { {"char1", "char(10)"}, }; HiveCharWritable[][] rows = new HiveCharWritable[][] { {charW("Hello", 10)}, {charW("world!", 10)}}; ArrowColumnarBatchSerDe serDe = new ArrowColumnarBatchSerDe(); StructObjectInspector rowOI = initSerDe(serDe, schema); ArrowWrapperWritable serialized = null; for (Object[] row : rows) { serialized = serDe.serialize(row, rowOI); } // Pass null to complete a batch if (serialized == null) { serialized = serDe.serialize(null, rowOI); } VarCharVector varCharVector = (VarCharVector) serialized.getVectorSchemaRoot().getFieldVectors().get(0); for (int i = 0; i < rows.length; i++) { assertEquals(rows[i][0].getPaddedValue().toString(), new String(varCharVector.get(i))); } }
static void populate(VarCharVector vector, byte[][] values){ vector.allocateNew(); Random r = new Random(); for(int i =0; i < values.length; i++){ byte[] val = values[i]; if(val != null){ vector.setSafe(i, val, 0, val.length); } else { // add noise. this confirms that after pivot, noise is gone. byte[] bytes = new byte[r.nextInt(15)]; r.nextBytes(bytes); vector.setSafe(i, bytes, 0, bytes.length); vector.setNull(i); } } vector.setValueCount(values.length); }
@Override public FieldVector getNewVector( String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new VarCharVector(name, fieldType, allocator); }
static String[] populateVarCharValuesWithoutNull(VarCharVector vector, int size){ vector.allocateNew(); String values[] = new String[size]; for(int i =0; i < values.length; i++){ values[i] = RandomStringUtils.randomAlphanumeric(RAND.nextInt(25)); vector.setSafe(i, values[i].getBytes(Charsets.UTF_8)); } vector.setValueCount(values.length); return values; }
@Override public boolean endField() { fieldOpen = false; if(collect) { assert currentVector != null; currentVector.setSafe(recordCount, fieldBytes, 0, currentDataPointer); } if (currentDataPointer > 0) { this.rowHasData = true; } return currentFieldIndex < maxField; }
private void decodeVarChar(IntVector input, VarCharVector output, VarBinaryVector dictionary) { if (hasSv2) { final SelectionVector2 sv2 = incoming.getSelectionVector2(); for (int i = 0; i < recordsConsumedCurrentBatch; ++i) { final int svIndex = sv2.getIndex(i); // TODO : Implement CopySafe between varchar and varbinary vectors. if (!input.isNull(svIndex)) { output.setNull(svIndex); } else { final int id = input.get(svIndex); final byte[] value = dictionary.get(id); output.setSafe(svIndex, value, 0, value.length); } } } else { for (int i = 0; i < recordsConsumedCurrentBatch; ++i) { if (input.isNull(i)) { output.setNull(i); } else { final int id = input.get(i); final byte[] value = dictionary.get(id); output.setSafe(i, value, 0, value.length); } } } }
public void writeNull() { vector.setNull(idx()); vector.setValueCount(idx()+1); } }
public void loadData(SerializedField metadata, ArrowBuf buffer) { /* clear the current buffers (if any) */ vector.clear(); /* get the metadata children */ final SerializedField bitsField = metadata.getChild(0); final SerializedField valuesField = metadata.getChild(1); final int valuesLength = buffer.capacity(); vector.allocateNew(valuesLength, metadata.getValueCount()); /* set inner validity buffer */ setValidityBuffer(bitsField); /* load inner offset and value buffers */ vector.offsetBuffer.close(); vector.valueBuffer.close(); loadOffsetAndDataBuffer(valuesField, buffer.slice(0, valuesLength)); vector.setLastSet(metadata.getValueCount() - 1); }
@Override public boolean setSafe(int index, ArrowBuf value, int start, int length) { if (index >= vector.getValueCapacity()) { return false; } if (usingDictionary) { ByteBuffer buf = currDictValToWrite.toByteBuffer(); vector.setSafe(index, buf, buf.position(), currDictValToWrite.length()); } else { vector.setSafe(index, 1, start, start + length, value); } return true; }
@Override public boolean isNull(int index) { return ac.isNull(index); }
@Override public void allocate() { vector.allocateNew(); }
switch (type.getMode()) { case REQUIRED: ((VarCharVector) vector).setSafe(index, (VarCharHolder) holder); return; case OPTIONAL: if (holder instanceof NullableVarCharHolder) { if (((NullableVarCharHolder) holder).isSet == 1) { ((VarCharVector) vector).setSafe(index, (NullableVarCharHolder) holder); } else { ((VarCharVector) vector).isSafe(index); ((VarCharVector) vector).setSafe(index, (VarCharHolder) holder);