private boolean nullFilled(ValueVector vector) { for (int r = 0; r < vector.getValueCount(); r++) { if (!vector.isNull(r)) { return false; } } return true; }
/** * Returns an instance sitting at the given index if exists, null otherwise. * * @see com.dremio.exec.vector.accessor.SqlAccessor#getObject(int) */ @Override public Object getObject(int rowOffset) throws InvalidAccessException { // In case some vectors have fewer values than others, and callee invokes // this method with index >= getValueCount(), this should still yield null. if (rowOffset < vector.getValueCount()) { return delegate.getObject(rowOffset); } return null; } }
public void determineTotalSize() { for (ValueVector vv : hyperVector.getValueVectors()) { this.totalValues += vv.getValueCount(); } }
public void printColumnMajor(ValueVector vv) { if (ParquetRecordReaderTest.VERBOSE_DEBUG){ System.out.println("\n" + vv.getField().getName()); } for (int j = 0; j < vv.getValueCount(); j++) { if (ParquetRecordReaderTest.VERBOSE_DEBUG){ Object o = vv.getObject(j); if (o instanceof byte[]) { try { o = new String((byte[])o, "UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } System.out.print(Strings.padStart(o + "", 20, ' ') + " "); System.out.print(", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : "")); } } if (ParquetRecordReaderTest.VERBOSE_DEBUG) { System.out.println("\n" + vv.getValueCount()); } }
@Override public boolean hasNext() { if (totalValuesRead == recordLimit) { return false; } if (indexInVectorList < hyperVector.getValueVectors().length) { return true; } else if ( indexInCurrentVector < currVec.getValueCount()) { return true; } return false; }
@Override public Object getObject(int index) { Map<String, Object> vv = new JsonStringHashMap<>(); for (String child : getChildFieldNames()) { ValueVector v = getChild(child); if (v != null && index < v.getValueCount()) { Object value = v.getObject(index); if (value != null) { vv.put(child, value); } } } return vv; }
@Override public Object next() { if (currVec == null || indexInCurrentVector == currVec.getValueCount()) { currVec = hyperVector.getValueVectors()[indexInVectorList]; indexInVectorList++; indexInCurrentVector = 0; } Object obj = currVec.getObject(indexInCurrentVector); indexInCurrentVector++; totalValuesRead++; return obj; }
/** * Decodes a dictionary encoded array using the provided dictionary. * * @param indices dictionary encoded values, must be int type * @param dictionary dictionary used to decode the values * @return vector with values restored from dictionary */ public static ValueVector decode(ValueVector indices, Dictionary dictionary) { int count = indices.getValueCount(); ValueVector dictionaryVector = dictionary.getVector(); int dictionaryCount = dictionaryVector.getValueCount(); // copy the dictionary values into the decoded vector TransferPair transfer = dictionaryVector.getTransferPair(indices.getAllocator()); transfer.getTo().allocateNewSafe(); for (int i = 0; i < count; i++) { Object index = indices.getObject(i); if (index != null) { int indexAsInt = ((Number) index).intValue(); if (indexAsInt > dictionaryCount) { throw new IllegalArgumentException("Provided dictionary does not contain value for index " + indexAsInt); } transfer.copyValueSafe(indexAsInt, i); } } // TODO do we need to worry about the field? ValueVector decoded = transfer.getTo(); decoded.setValueCount(count); return decoded; }
@Override public int outputData() { final int count = incoming.getRecordCount(); if (randomVector == null) { // if nothing is projected, just set the count and return } else if (straightCopy || count == randomVector.getValueCount()){ for(TransferPair tp : transferPairs){ tp.transfer(); } } else { final long addr = sv2.memoryAddress(); for (FieldBufferCopier copier : copiers) { copier.copy(addr, count); } } state = State.CAN_CONSUME; output.setAllCount(count); return count; }
@Test public void simpleEqualityJoin() throws Throwable { // Function checks hash join with single equality condition final String plan = Files.toString(FileUtils.getResourceAsFile("/join/hash_join.json"), Charsets.UTF_8) .replace("#{TEST_FILE_1}", FileUtils.getResourceAsFile("/build_side_input.json").toURI().toString()) .replace("#{TEST_FILE_2}", FileUtils.getResourceAsFile("/probe_side_input.json").toURI().toString()); List<QueryDataBatch> results = testRunAndReturn(QueryType.PHYSICAL, plan); try(RecordBatchLoader batchLoader = new RecordBatchLoader(nodes[0].getContext().getAllocator())){ QueryDataBatch batch = results.get(1); assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData())); Iterator<VectorWrapper<?>> itr = batchLoader.iterator(); // Just test the join key long colA[] = {1, 1, 2, 2, 1, 1}; // Check the output of decimal9 ValueVector intValueVector = itr.next().getValueVector(); for (int i = 0; i < intValueVector.getValueCount(); i++) { assertEquals(intValueVector.getObject(i), colA[i]); } assertEquals(6, intValueVector.getValueCount()); } for (QueryDataBatch result : results) { result.release(); } }
protected Object[] getRunResult(QueryType queryType, String planString) throws Exception { List<QueryDataBatch> resultList = testRunAndReturn(queryType, planString); List<Object> res = new ArrayList<>(); RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); for(QueryDataBatch result : resultList) { if (result.getData() != null) { loader.load(result.getHeader().getDef(), result.getData()); ValueVector v = loader.iterator().next().getValueVector(); for (int j = 0; j < v.getValueCount(); j++) { if (v instanceof VarCharVector) { res.add(new String(((VarCharVector) v).get(j))); } else { res.add(v.getObject(j)); } } loader.clear(); result.release(); } } return res.toArray(); }
@Test public void multipleConditionJoin() throws Exception { // Function tests hash join with multiple join conditions final String plan = Files .toString(FileUtils.getResourceAsFile("/join/hj_multi_condition_join.json"), Charsets.UTF_8) .replace("#{TEST_FILE_1}", FileUtils.getResourceAsFile("/build_side_input.json").toURI().toString()) .replace("#{TEST_FILE_2}", FileUtils.getResourceAsFile("/probe_side_input.json").toURI().toString()); List<QueryDataBatch> results = testRunAndReturn(QueryType.PHYSICAL, plan); try (RecordBatchLoader batchLoader = new RecordBatchLoader(nodes[0].getContext().getAllocator())) { final QueryDataBatch batch = results.get(1); assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData())); final Iterator<VectorWrapper<?>> itr = batchLoader.iterator(); // Just test the join key final long colA[] = { 1, 2, 1 }; final long colC[] = { 100, 200, 500 }; // Check the output of decimal9 final ValueVector intValueVector1 = itr.next().getValueVector(); final ValueVector intValueVector2 = itr.next().getValueVector(); for (int i = 0; i < intValueVector1.getValueCount(); i++) { assertEquals(intValueVector1.getObject(i), colA[i]); assertEquals(intValueVector2.getObject(i), colC[i]); } assertEquals(3, intValueVector1.getValueCount()); } for (final QueryDataBatch result : results) { result.release(); } }
protected Object[] getRunResult(QueryType queryType, String planString) throws Exception { List<QueryDataBatch> resultList = testRunAndReturn(queryType, planString); List<Object> res = new ArrayList<>(); try(RecordBatchLoader loader = new RecordBatchLoader(getAllocator())){; for(QueryDataBatch result : resultList) { if (result.getData() != null) { loader.load(result.getHeader().getDef(), result.getData()); ValueVector v = loader.iterator().next().getValueVector(); for (int j = 0; j < v.getValueCount(); j++) { res.add(v.getObject(j)); } loader.clear(); result.release(); } } } return res.toArray(); }
@Override public int outputData() { if(checkForStraightCopy && incoming.getRecordCount() == randomVector.getValueCount()){ for(TransferPair tp : transferPairs){ tp.transfer(); } output.setRecordCount(incoming.getRecordCount()); state = State.CAN_CONSUME; return incoming.getRecordCount(); } int recordCount = incoming.getRecordCount() - this.copyOffset; int copiedRecords = copier.copyRecords(copyOffset, recordCount); if(copiedRecords < recordCount){ copyOffset = copyOffset + copiedRecords; }else{ copyOffset = 0; state = State.CAN_CONSUME; } if(copiedRecords > 0){ for(VectorWrapper<?> v : output){ v.getValueVector().setValueCount(copiedRecords); } } output.setRecordCount(copiedRecords); return copiedRecords; }
for (int i = 0; i < dec9ValueVector.getValueCount(); i++) { assertEquals(dec9ValueVector.getObject(i).toString(), decimal9Output[i]); assertEquals(dec38ValueVector.getObject(i).toString(), decimal38Output[i]); assertEquals(6, dec9ValueVector.getValueCount()); assertEquals(6, dec38ValueVector.getValueCount());
System.out.print(vw.getValueVector().getField().getName() + ": "); ValueVector vv = vw.getValueVector(); for (int i = 0; i < vv.getValueCount(); i++) { Object o = vv.getObject(i); if (o instanceof byte[]) {
@Override public void dataArrived(QueryDataBatch result, ConnectionThrottle throttle) { QueryData queryHeader = result.getHeader(); int rows = queryHeader.getRowCount(); try { if ( result.hasData() ) { ArrowBuf data = result.getData(); loader.load(queryHeader.getDef(), data); for (int i = 0; i < rows; i++) { Map<String,String> record = Maps.newHashMap(); for (VectorWrapper<?> vw : loader) { final String field = vw.getValueVector().getField().getName(); final ValueVector vv = vw.getValueVector(); final Object value = i < vv.getValueCount() ? vv.getObject(i) : null; final String display = value == null ? null : value.toString(); record.put(field, display); } records.add(record); } loader.clear(); } result.release(); } catch (SchemaChangeException e) { fail(e.getMessage()); } }
for (int i = 0; i < vv.getValueCount(); i++) { final Object o = vv.getObject(i); builder.append(o);
ValueVector varcharValueVector = itr.next().getValueVector(); for (int i = 0; i < intValueVector.getValueCount(); i++) { System.out.println(intValueVector.getObject(i)); assertEquals(intValueVector.getObject(i), 10);
@Test @Ignore("decimal") public void testSimpleDecimalMathFunc() throws Exception { try (ClusterCoordinator clusterCoordinator = LocalClusterCoordinator.newRunningCoordinator(); SabotNode bit = new SabotNode(DEFAULT_SABOT_CONFIG, clusterCoordinator, CLASSPATH_SCAN_RESULT, true); DremioClient client = new DremioClient(DEFAULT_SABOT_CONFIG, clusterCoordinator)) { // run query. bit.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(com.dremio.exec.proto.UserBitShared.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/decimal/simple_decimal_math.json"), Charsets.UTF_8) .replace("#{TEST_FILE}", "/input_simple_decimal.json") ); RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator()); QueryDataBatch batch = results.get(0); assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData())); Iterator<VectorWrapper<?>> itr = batchLoader.iterator(); // Check the output of decimal18 ValueVector dec18ValueVector = itr.next().getValueVector(); assertEquals(6, dec18ValueVector.getValueCount()); batchLoader.clear(); for (QueryDataBatch result : results) { result.release(); } } }