@After public void after() throws Exception { this.cleanupTestMetadata(); }
@Before public void setup() throws IOException { this.createTestMetadata(); table = newTestTable(); info = table.getInfo(); timeComp0 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-14")); timeComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14")); timeComp2 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-13")); timeComp3 = compare(info.colRef(0), FilterOperatorEnum.LT, enc(info, 0, "2015-01-15")); timeComp4 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-15")); timeComp5 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-15")); timeComp6 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "2015-01-14")); timeComp7 = compare(info.colRef(0), FilterOperatorEnum.EQ, enc(info, 0, "1970-01-01")); ageComp1 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "10")); ageComp2 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "20")); ageComp3 = compare(info.colRef(1), FilterOperatorEnum.EQ, enc(info, 1, "30")); ageComp4 = compare(info.colRef(1), FilterOperatorEnum.NEQ, enc(info, 1, "30")); }
public static LogicalTupleFilter and(TupleFilter... children) { return logic(FilterOperatorEnum.AND, children); }
private static CubeCodeSystem newDictCodeSystem() { DimensionEncoding[] dimEncs = new DimensionEncoding[3]; dimEncs[1] = new DictionaryDimEnc(newDictionaryOfInteger()); dimEncs[2] = new DictionaryDimEnc(newDictionaryOfString()); return new CubeCodeSystem(dimEncs); }
@Test public void verifyScanWithUnevaluatableFilter() throws IOException { GTInfo info = table.getInfo(); CompareTupleFilter fComp = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14")); ExtractTupleFilter fUnevaluatable = unevaluatable(info.colRef(1)); LogicalTupleFilter fNotPlusUnevaluatable = not(unevaluatable(info.colRef(1))); LogicalTupleFilter filter = and(fComp, fUnevaluatable, fNotPlusUnevaluatable); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" }) .setFilterPushDown(filter).createGTScanRequest(); // note the unEvaluatable column 1 in filter is added to group by assertEquals( "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], [null], [null]], aggrGroupBy={0, 1}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString()); doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 20, null]", "[1421280000000, 30, null, 10, null]", "[1421366400000, 20, null, 20, null]", "[1421366400000, 30, null, 20, null]", "[1421452800000, 10, null, 10, null]"); }
@Test public void verifyScanWithEvaluatableFilter() throws IOException { GTInfo info = table.getInfo(); CompareTupleFilter fComp1 = compare(info.colRef(0), FilterOperatorEnum.GT, enc(info, 0, "2015-01-14")); CompareTupleFilter fComp2 = compare(info.colRef(1), FilterOperatorEnum.GT, enc(info, 1, "10")); LogicalTupleFilter filter = and(fComp1, fComp2); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setAggrGroupBy(setOf(0)).setAggrMetrics(setOf(3)).setAggrMetricsFuncs(new String[] { "sum" }) .setFilterPushDown(filter).createGTScanRequest(); // note the evaluatable column 1 in filter is added to returned columns but not in group by assertEquals( "GTScanRequest [range=[[null, null]-[null, null]], columns={0, 1, 3}, filterPushDown=AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GT [\\x00]], aggrGroupBy={0}, aggrMetrics={3}, aggrMetricsFuncs=[sum]]", req.toString()); doScanAndVerify(table, useDeserializedGTScanRequest(req), "[1421280000000, 20, null, 30, null]", "[1421366400000, 20, null, 40, null]"); }
@Test public void verifyAggregateAndHavingFilter() throws IOException { GTInfo info = table.getInfo(); TblColRef havingCol = TblColRef.newInnerColumn("SUM_OF_BIGDECIMAL", InnerDataTypeEnum.LITERAL); havingCol.getColumnDesc().setId("1"); // point to the first aggregated measure CompareTupleFilter havingFilter = compare(havingCol, FilterOperatorEnum.GT, "20"); GTScanRequest req = new GTScanRequestBuilder().setInfo(info).setRanges(null).setDimensions(null) .setAggrGroupBy(setOf(1)).setAggrMetrics(setOf(4)).setAggrMetricsFuncs(new String[] { "sum" }) .setHavingFilterPushDown(havingFilter).createGTScanRequest(); doScanAndVerify(table, useDeserializedGTScanRequest(req), "[null, 20, null, null, 42.0]", "[null, 30, null, null, 52.5]"); }
@Test public void verifyConvertFilterConstants1() { GTInfo info = table.getInfo(); TableDesc extTable = TableDesc.mockup("ext"); TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp"); TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer"); CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.EQ, "10"); LogicalTupleFilter filter = and(fComp1, fComp2); List<TblColRef> colMapping = Lists.newArrayList(); colMapping.add(extColA); colMapping.add(extColB); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals( "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 EQ [\\x00]]", newFilter.toString()); }
ByteArray segmentStart = enc(info, 0, "2015-01-14"); ByteArray segmentStartX = enc(info, 0, "2015-01-14 00:00:00");//when partition col is dict encoded, time format will be free assertEquals(segmentStart, segmentStartX); LogicalTupleFilter filter = and(timeComp0, ageComp1); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); LogicalTupleFilter filter = and(timeComp2, ageComp1); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); LogicalTupleFilter filter = and(timeComp4, ageComp1); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); LogicalTupleFilter filter = and(timeComp5, ageComp1); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); LogicalTupleFilter filter = or(and(timeComp2, ageComp1), and(timeComp1, ageComp1), and(timeComp6, ageComp1)); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); LogicalTupleFilter filter = or(and(timeComp3, ageComp3), and(timeComp7, ageComp1)); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, filter); List<GTScanRange> r = planner.planScanRanges();
static GTInfo newInfo() { Builder builder = GTInfo.builder(); builder.setCodeSystem(newDictCodeSystem()); builder.setColumns(// DataType.getType("timestamp"), // DataType.getType("integer"), // DataType.getType("varchar(10)"), // DataType.getType("bigint"), // DataType.getType("decimal") // ); builder.setPrimaryKey(setOf(0, 1)); builder.setColumnPreferIndex(setOf(0)); builder.enableColumnBlock(new ImmutableBitSet[] { setOf(0, 1), setOf(2), setOf(3, 4) }); builder.enableRowBlock(4); GTInfo info = builder.build(); return info; }
LogicalTupleFilter filter = and(timeComp1, or(ageComp1, ageComp2)); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, filter); List<GTScanRange> r = planner.planScanRanges(); LogicalTupleFilter filter = and(timeComp1, timeComp2); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, filter); List<GTScanRange> r = planner.planScanRanges(); LogicalTupleFilter filter = or(timeComp1, ageComp4); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, filter); List<GTScanRange> r = planner.planScanRanges(); LogicalTupleFilter filter = or(timeComp1, timeComp3); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, filter); List<GTScanRange> r = planner.planScanRanges(); LogicalTupleFilter filter = or(and(timeComp4, ageComp1), and(timeComp4, ageComp2), and(timeComp4, ageComp3)); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, null, filter); List<GTScanRange> r = planner.planScanRanges();
@Test public void verifySegmentSkipping2() { { LogicalTupleFilter filter = and(timeComp0, ageComp1); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(1, r.size());//scan range are [close,close] assertEquals("[null, 10]-[1421193600000, 10]", r.get(0).toString()); assertEquals(1, r.get(0).fuzzyKeys.size()); assertEquals("[[null, 10, null, null, null]]", r.get(0).fuzzyKeys.toString()); } { LogicalTupleFilter filter = and(timeComp5, ageComp1); CubeScanRangePlanner planner = new CubeScanRangePlanner(info, info.colRef(0), filter); List<GTScanRange> r = planner.planScanRanges(); assertEquals(1, r.size());//scan range are [close,close] } }
@Test public void verifyFirstRow() throws IOException { doScanAndVerify(table, new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null) .setFilterPushDown(null).createGTScanRequest(), "[1421193600000, 30, Yang, 10, 10.5]", // "[1421193600000, 30, Luke, 10, 10.5]", // "[1421280000000, 20, Dong, 10, 10.5]", // "[1421280000000, 20, Jason, 10, 10.5]", // "[1421280000000, 30, Xu, 10, 10.5]", // "[1421366400000, 20, Mahone, 10, 10.5]", // "[1421366400000, 20, Qianhao, 10, 10.5]", // "[1421366400000, 30, George, 10, 10.5]", // "[1421366400000, 30, Shaofeng, 10, 10.5]", // "[1421452800000, 10, Kejia, 10, 10.5]"); }
@Test public void verifyConvertFilterConstants4() { GTInfo info = table.getInfo(); TableDesc extTable = TableDesc.mockup("ext"); TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp"); TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer"); CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.IN, "9", "10", "15"); LogicalTupleFilter filter = and(fComp1, fComp2); List<TblColRef> colMapping = Lists.newArrayList(); colMapping.add(extColA); colMapping.add(extColB); // $1 in ("9", "10", "15") has only "10" left TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals( "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 IN [\\x00]]", newFilter.toString()); }
colMapping.add(extColB); CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GT, "101")); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals("AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 ISNOTNULL []]", LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GT, "100")); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals( LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GT, "99")); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals( LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GTE, "101")); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals(ConstantTupleFilter.FALSE, newFilter); LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GTE, "100")); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals( LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GTE, "99")); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals(
public static LogicalTupleFilter or(TupleFilter... children) { return logic(FilterOperatorEnum.OR, children); }
colMapping.add(extColB); CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14"); LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LT, "9")); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals( LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LT, "10")); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals( LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LT, "11")); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals( LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LTE, "9")); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals(ConstantTupleFilter.FALSE, newFilter); LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LTE, "10")); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals( LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LTE, "11")); TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null); assertEquals(
public static LogicalTupleFilter not(TupleFilter child) { return logic(FilterOperatorEnum.NOT, child); }