public Set<TblColRef> getShardByColumns() { return getRowkey().getShardByColumns(); }
public static long getBaseCuboidId(CubeDesc cube) { return cube.getRowkey().getFullMask(); }
private long getQueryFilterMask(Set<TblColRef> filterColumnD) { long filterMask = 0; logger.info("Filter column set for query: %s", filterColumnD); if (filterColumnD.isEmpty() == false) { RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { if (filterColumnD.contains(allColumns[i].getColRef())) { filterMask |= 1L << allColumns[i].getBitIndex(); } } } logger.info("Filter mask is: {0}", filterMask); return filterMask; }
private List<TblColRef> translateIdToColumns(long cuboidID) { List<TblColRef> dimesnions = new ArrayList<TblColRef>(); RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { // NOTE: the order of column in list!!! long bitmask = 1L << allColumns[i].getBitIndex(); if ((cuboidID & bitmask) != 0) { TblColRef colRef = allColumns[i].getColRef(); dimesnions.add(colRef); } } return dimesnions; }
public int getRowKeyColumnCount() { return getDescriptor().getRowkey().getRowKeyColumns().length; }
public TblColRef getColumnByBitIndex(int bitIndex) { RowKeyColDesc[] rowKeyColumns = this.getRowkey().getRowKeyColumns(); return rowKeyColumns[rowKeyColumns.length - 1 - bitIndex].getColRef(); }
public Set<Long> generateMandatoryCuboids(List<Set<String>> mandatoryDimensionSetList) { Map<String, RowKeyColDesc> rowKeyColDescMap = Maps.newHashMap(); for (RowKeyColDesc entry : getRowkey().getRowKeyColumns()) { rowKeyColDescMap.put(entry.getColumn(), entry); } Set<Long> mandatoryCuboids = Sets.newHashSetWithExpectedSize(mandatoryDimensionSetList.size()); for (Set<String> mandatoryDimensionSet : mandatoryDimensionSetList) { long cuboid = 0L; for (String columnName : mandatoryDimensionSet) { TblColRef tblColRef = model.findColumn(columnName); RowKeyColDesc rowKeyColDesc = rowKeyColDescMap.get(tblColRef.getIdentity()); // check if mandatory dimension set list is valid if (rowKeyColDesc == null) { logger.warn("Column " + columnName + " in " + mandatoryDimensionSet + " does not exist"); throw new IllegalStateException( "Column " + columnName + " in " + mandatoryDimensionSet + " does not exist"); } cuboid |= 1L << rowKeyColDesc.getBitIndex(); } mandatoryCuboids.add(cuboid); } return mandatoryCuboids; }
RowKeyColDesc[] rowKeyColDescList = cube.getDescriptor().getRowkey().getRowKeyColumns();
public DefaultCuboidScheduler(CubeDesc cubeDesc) { super(cubeDesc); int size = this.cubeDesc.getRowkey().getRowKeyColumns().length; this.max = (long) Math.pow(2, size) - 1; Pair<Set<Long>, Map<Long, List<Long>>> pair = buildTreeBottomUp(); this.allCuboidIds = pair.getFirst(); this.parent2child = pair.getSecond(); }
public static long toCuboidId(CubeDesc cubeDesc, Set<TblColRef> dimensions, Collection<FunctionDesc> metrics) { for (FunctionDesc metric : metrics) { if (metric.getMeasureType().onlyAggrInBaseCuboid()) return Cuboid.getBaseCuboidId(cubeDesc); } long cuboidID = 0; for (TblColRef column : dimensions) { int index = cubeDesc.getRowkey().getColumnBitIndex(column); cuboidID |= 1L << index; } return cuboidID; }
public RowKeySplitter(CubeSegment cubeSeg) { this(cubeSeg, cubeSeg.getCubeDesc().getRowkey().getRowKeyColumns().length + 2, cubeSeg.getConfig().getDimensionEncodingMaxLength()); } public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
private void parseCubeDesc() { Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc); // build index for rowkey columns List<TblColRef> cuboidColumns = baseCuboid.getColumns(); int rowkeyColCount = cubeDesc.getRowkey().getRowKeyColumns().length; rowKeyColumnIndexes = new int[rowkeyColCount]; for (int i = 0; i < rowkeyColCount; i++) { TblColRef col = cuboidColumns.get(i); rowKeyColumnIndexes[i] = flatDesc.getColumnIndex(col); } List<MeasureDesc> measures = cubeDesc.getMeasures(); int measureSize = measures.size(); measureColumnIndexes = new int[measureSize][]; for (int i = 0; i < measureSize; i++) { FunctionDesc func = measures.get(i).getFunction(); List<TblColRef> colRefs = func.getParameter().getColRefs(); if (colRefs == null) { measureColumnIndexes[i] = null; } else { measureColumnIndexes[i] = new int[colRefs.size()]; for (int j = 0; j < colRefs.size(); j++) { TblColRef c = colRefs.get(j); measureColumnIndexes[i][j] = flatDesc.getColumnIndex(c); } } } }
@Override public void validate(CubeDesc cube, ValidateContext context) { RowKeyDesc row = cube.getRowkey(); if (row == null) { context.addResult(ResultLevel.ERROR, "Rowkey does not exist"); return; } RowKeyColDesc[] rcd = row.getRowKeyColumns(); if (rcd == null || rcd.length == 0) { context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist"); return; } for (int i = 0; i < rcd.length; i++) { RowKeyColDesc rd = rcd[i]; if (rd.getColumn() == null || rd.getColumn().length() == 0) { context.addResult(ResultLevel.ERROR, "Rowkey column empty"); } } } }
public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) { final String tableName = flatDesc.getTableName(); StringBuilder sql = new StringBuilder(); sql.append("INSERT OVERWRITE TABLE " + quoteIdentifier(tableName) + " SELECT * FROM " + quoteIdentifier(tableName)); if (flatDesc.getClusterBy() != null) { appendClusterStatement(sql, flatDesc.getClusterBy()); } else if (flatDesc.getDistributedBy() != null) { appendDistributeStatement(sql, Lists.newArrayList(flatDesc.getDistributedBy())); } else { int redistColumnCount = cubeDesc.getConfig().getHiveRedistributeColumnCount(); RowKeyColDesc[] rowKeyColDescs = cubeDesc.getRowkey().getRowKeyColumns(); if (rowKeyColDescs.length < redistColumnCount) redistColumnCount = rowKeyColDescs.length; List<TblColRef> redistColumns = Lists.newArrayListWithCapacity(redistColumnCount); for (int i = 0; i < redistColumnCount; i++) { redistColumns.add(rowKeyColDescs[i].getColRef()); } appendDistributeStatement(sql, redistColumns); } return sql.toString(); }
@Override public DimensionEncoding get(TblColRef col) { DimensionEncoding result = encMap.get(col); if (result == null) { RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col); if (colDesc.isUsingDictionary()) { // special dictionary encoding Dictionary<String> dict = getDictionary(col); if (dict == null) { logger.warn("No dictionary found for dict-encoding column " + col + ", segment " + seg); result = new FixedLenDimEnc(0); } else { result = new DictionaryDimEnc(dict); } } else { // normal case result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion()); } encMap.put(col, result); } return result; }
nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) { this.enableSharding = cubeSeg.isEnableSharding(); this.cubeDesc = cubeSeg.getCubeDesc(); IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg); for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) { dimEncoding.get(rowKeyColDesc.getColRef()); } this.colIO = new RowKeyColumnIO(dimEncoding); this.splitBuffers = new ByteArray[splitLen]; this.splitOffsets = new int[splitLen]; this.bufferSize = 0; }
@Test public void testIdentifyCuboidV2() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, NoSuchFieldException { CubeDesc cubeDesc = cube.getDescriptor(); Cuboid ret = Cuboid.findCuboid(cube.getCuboidScheduler(), Sets.<TblColRef> newHashSet(), Lists.<FunctionDesc> newArrayList()); long baseCuboidId = cubeDesc.getRowkey().getFullMask(); assertNotEquals(baseCuboidId, ret.getId()); ret = Cuboid.findCuboid(cube.getCuboidScheduler(), dimensions, metrics); assertEquals(baseCuboidId, ret.getId()); }
@Test public void testBadDesc() throws IOException { ValidateContext vContext = new ValidateContext(); CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); desc.getRowkey().getRowKeyColumns()[2].setColumn(""); IValidatorRule<CubeDesc> rule = new RowKeyAttrRule(); rule.validate(desc, vContext); vContext.print(System.out); assertTrue(vContext.getResults().length == 1); assertTrue("Rowkey column empty".equalsIgnoreCase(vContext.getResults()[0].getMessage())); } }
newCubeDesc.setMeasures(cubeDesc.getMeasures()); newCubeDesc.setDictionaries(cubeDesc.getDictionaries()); newCubeDesc.setRowkey(cubeDesc.getRowkey()); newCubeDesc.setHbaseMapping(cubeDesc.getHbaseMapping()); newCubeDesc.setSignature(cubeDesc.getSignature());