/** * Called before processing fields. This method fills any fields that have * NULL values or have type NULL_TYPE with a NullDatum. */ @Override public void start() { currentTuple = new VTuple(tupleSize); }
public VTuple(Datum [] datum) { this(datum.length); put(datum); }
public void analyzeField(int idx, Tuple tuple) { if (columnStatsEnabled[idx]) { if (tuple.isBlankOrNull(idx)) { numNulls[idx]++; return; } Datum datum = tuple.asDatum(idx); if (!maxValues.contains(idx) || maxValues.get(idx).compareTo(datum) < 0) { maxValues.put(idx, datum); } if (!minValues.contains(idx) || minValues.get(idx).compareTo(datum) > 0) { minValues.put(idx, datum); } } }
@Test public void testPartitionForOnePartNum() { Schema schema = SchemaBuilder.builder() .add("l_returnflag", Type.TEXT) .add("l_linestatus", Type.TEXT) .build(); SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); VTuple s = new VTuple(2); s.put(0, DatumFactory.createText("A")); s.put(1, DatumFactory.createText("F")); VTuple e = new VTuple(2); e.put(0, DatumFactory.createText("R")); e.put(1, DatumFactory.createText("O")); TupleRange expected = new TupleRange(sortSpecs, s, e); RangePartitionAlgorithm partitioner = new UniformRangePartition(expected, sortSpecs, true); TupleRange [] ranges = partitioner.partition(1); assertEquals(expected, ranges[0]); }
@Test public void testClone() throws CloneNotSupportedException { VTuple t1 = new VTuple(5); t1.put(0, DatumFactory.createInt4(1)); t1.put(1, DatumFactory.createInt4(2)); t1.put(3, DatumFactory.createInt4(2)); t1.put(4, DatumFactory.createText("str")); VTuple t2 = t1.clone(); assertNotSame(t1, t2); assertEquals(t1, t2); assertSame(t1.get(4), t2.get(4)); t1.clear(); assertFalse(t1.equals(t2)); } }
@Test public void testPut() { VTuple t1 = new VTuple(260); t1.put(0, DatumFactory.createText("str")); t1.put(1, DatumFactory.createInt4(2)); t1.put(257, DatumFactory.createFloat4(0.76f)); assertTrue(t1.contains(0)); assertTrue(t1.contains(1)); assertEquals(t1.getText(0),"str"); assertEquals(t1.get(1).asInt4(),2); assertTrue(t1.get(257).asFloat4() == 0.76f); }
@Test public void testContain() { VTuple t1 = new VTuple(260); t1.put(0, DatumFactory.createInt4(1)); t1.put(1, DatumFactory.createInt4(1)); t1.put(27, DatumFactory.createInt4(1)); t1.put(96, DatumFactory.createInt4(1)); t1.put(257, DatumFactory.createInt4(1)); assertTrue(t1.contains(0)); assertTrue(t1.contains(1)); assertFalse(t1.contains(2)); assertFalse(t1.contains(3)); assertFalse(t1.contains(4)); assertTrue(t1.contains(27)); assertFalse(t1.contains(28)); assertFalse(t1.contains(95)); assertTrue(t1.contains(96)); assertFalse(t1.contains(97)); assertTrue(t1.contains(257)); }
appender.init(); VTuple tuple = new VTuple(index - 1); index = 0; if (dateTypeSupport()) { tuple.put(index++, DatumFactory.createDate("1980-04-01")); tuple.put(index++, DatumFactory.createTimestampDatumWithUnixTime((int)(System.currentTimeMillis() / 1000))); tuple.put(index, DatumFactory.createTime("12:34:56")); for (int i = 0; i < tuple.size(); i++) { assertEquals("failed at " + i + " th column", tuple.get(i), retrieved.asDatum(i));
@Test public void testDateTextHandling() throws Exception { if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO) || internalType) { return; } Schema schema = SchemaBuilder.builder().add("col1", Type.TEXT).build(); TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf); FileTablespace sm = TablespaceManager.getLocalFs(); Path tablePath = new Path(testDir, "testTextHandling.data"); Appender appender = sm.getAppender(meta, schema, tablePath); appender.init(); VTuple tuple = new VTuple(1); tuple.put(0, DatumFactory.createDate(1994,7,30)); appender.addTuple(tuple); appender.flush(); appender.close(); FileStatus status = fs.getFileStatus(tablePath); FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); Scanner scanner = sm.getScanner(meta, schema, fragment, null); scanner.init(); Tuple retrieved; while ((retrieved = scanner.next()) != null) { assertEquals(tuple.get(0).asChars(), retrieved.asDatum(0).asChars()); } scanner.close(); if (internalType){ OldStorageManager.clearCache(); } }
public TableStats getTableStat() { TableStats stat = new TableStats(); for (int i = 0; i < schema.size(); i++) { if (columnStatsEnabled[i]) { Column column = schema.getColumn(i); ColumnStats columnStats = new ColumnStats(column); columnStats.setNumNulls(numNulls[i]); if (minValues.isBlank(i) || column.getDataType().getType() == minValues.type(i)) { columnStats.setMinValue(minValues.get(i)); } else { LOG.warn("Wrong statistics column type (" + minValues.type(i) + ", expected=" + column.getDataType().getType() + ")"); } if (minValues.isBlank(i) || column.getDataType().getType() == maxValues.type(i)) { columnStats.setMaxValue(maxValues.get(i)); } else { LOG.warn("Wrong statistics column type (" + maxValues.type(i) + ", expected=" + column.getDataType().getType() + ")"); } stat.addColumnStat(columnStats); } } stat.setNumRows(this.numRows); stat.setNumBytes(this.numBytes); return stat; } }
@Override public void clear() { super.clear(); updateHashCode(); }
@Override public KeyTuple clone() throws CloneNotSupportedException { return (KeyTuple) super.clone(); }
@Test public void testPartitionForOnePartNumWithOneOfTheValueNull() { Schema schema = SchemaBuilder.builder() .add("l_returnflag", Type.TEXT) .add("l_linestatus", Type.TEXT) .build(); SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); VTuple s = new VTuple(2); s.put(0, DatumFactory.createNullDatum()); s.put(1, DatumFactory.createText("F")); VTuple e = new VTuple(2); e.put(0, DatumFactory.createText("R")); e.put(1, DatumFactory.createNullDatum()); TupleRange expected = new TupleRange(sortSpecs, s, e); RangePartitionAlgorithm partitioner = new UniformRangePartition(expected, sortSpecs, true); TupleRange [] ranges = partitioner.partition(1); assertEquals(expected, ranges[0]); }
public TableStats getTableStat() { TableStats stat = new TableStats(); for (int i = 0; i < schema.size(); i++) { if (columnStatsEnabled[i]) { Column column = schema.getColumn(i); ColumnStats columnStats = new ColumnStats(column); columnStats.setNumNulls(numNulls[i]); if (minValues.isBlank(i) || column.getDataType().getType() == minValues.type(i)) { columnStats.setMinValue(minValues.get(i)); } else { LOG.warn("Wrong statistics column type (" + minValues.type(i) + ", expected=" + column.getDataType().getType() + ")"); } if (minValues.isBlank(i) || column.getDataType().getType() == maxValues.type(i)) { columnStats.setMaxValue(maxValues.get(i)); } else { LOG.warn("Wrong statistics column type (" + maxValues.type(i) + ", expected=" + column.getDataType().getType() + ")"); } stat.addColumnStat(columnStats); } } stat.setNumRows(this.numRows); stat.setNumBytes(this.numBytes); return stat; } }
@Override public void clear() { super.clear(); updateHashCode(); }
@Override public KeyTuple clone() throws CloneNotSupportedException { return (KeyTuple) super.clone(); }
@Test public void testPartitionForOnePartNumWithBothValueNull() { Schema schema = SchemaBuilder.builder() .add("l_returnflag", Type.TEXT) .add("l_linestatus", Type.TEXT) .build(); SortSpec [] sortSpecs = PlannerUtil.schemaToSortSpecs(schema); VTuple s = new VTuple(2); s.put(0, DatumFactory.createNullDatum()); s.put(1, DatumFactory.createNullDatum()); VTuple e = new VTuple(2); e.put(0, DatumFactory.createNullDatum()); e.put(1, DatumFactory.createNullDatum()); TupleRange expected = new TupleRange(sortSpecs, s, e); RangePartitionAlgorithm partitioner = new UniformRangePartition(expected, sortSpecs, true); TupleRange [] ranges = partitioner.partition(1); assertEquals(expected, ranges[0]); }