private String getFilter(CubeMetastoreClient client, String tableName, String values) throws LensException { List<FieldSchema> cols = client.getHiveTable(tableName).getPartCols(); String[] vals = StringUtils.split(values, ","); if (vals.length != cols.size()) { log.error("Values for all the part columns not specified, cols:" + cols + " vals:" + Arrays.toString(vals)); throw new BadRequestException("Values for all the part columns not specified"); } StringBuilder filter = new StringBuilder(); for (int i = 0; i < vals.length; i++) { filter.append(cols.get(i).getName()); filter.append("="); filter.append("\""); filter.append(vals[i]); filter.append("\""); if (i != (vals.length - 1)) { filter.append(" AND "); } } return filter.toString(); }
public List<Partition> getAllParts(String storageTableName) throws HiveException, LensException { return getClient().getPartitions(getHiveTable(storageTableName)); }
public List<Partition> getAllParts(String storageTableName) throws HiveException, LensException { return getClient().getPartitions(getHiveTable(storageTableName)); }
private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, String> nonTimeParts, Map<String, TreeSet<Date>> timePartSpecs, UpdatePeriod updatePeriod) throws HiveException, LensException { Table hiveTable = getHiveTable(storageTableName); String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS); if (timePartColsStr != null) { LatestInfo latest = new LatestInfo(); String[] timePartCols = StringUtils.split(timePartColsStr, ','); for (String partCol : timePartCols) { if (!timePartSpecs.containsKey(partCol)) { continue; } boolean makeLatest = true; Partition part = getLatestPart(storageTableName, partCol, nonTimeParts); Date pTimestamp = timePartSpecs.get(partCol).last(); Date latestTimestamp = getLatestTimeStampFromPartition(part, partCol); if (latestTimestamp != null && pTimestamp.before(latestTimestamp)) { makeLatest = false; } if (makeLatest) { Map<String, String> latestParams = LensUtil.getHashMap(getLatestPartTimestampKey(partCol), updatePeriod.format(pTimestamp)); latest.latestParts.put(partCol, new LatestPartColumnInfo(latestParams)); } } return latest; } else { return null; } }
private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, String> nonTimeParts, Map<String, TreeSet<Date>> timePartSpecs, UpdatePeriod updatePeriod) throws HiveException, LensException { Table hiveTable = getHiveTable(storageTableName); String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS); if (timePartColsStr != null) { LatestInfo latest = new LatestInfo(); String[] timePartCols = StringUtils.split(timePartColsStr, ','); for (String partCol : timePartCols) { if (!timePartSpecs.containsKey(partCol)) { continue; } boolean makeLatest = true; Partition part = getLatestPart(storageTableName, partCol, nonTimeParts); Date pTimestamp = timePartSpecs.get(partCol).last(); Date latestTimestamp = getLatestTimeStampFromPartition(part, partCol); if (latestTimestamp != null && pTimestamp.before(latestTimestamp)) { makeLatest = false; } if (makeLatest) { Map<String, String> latestParams = LensUtil.getHashMap(getLatestPartTimestampKey(partCol), updatePeriod.format(pTimestamp)); latest.latestParts.put(partCol, new LatestPartColumnInfo(latestParams)); } } return latest; } else { return null; } }
public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); if (!dimTable.getStorages().isEmpty()) { for (String storageName : dimTable.getStorages()) { XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } dt.getStorageTables().getStorageTable().add(tblElement); } } return dt; }
public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); if (!dimTable.getStorages().isEmpty()) { for (String storageName : dimTable.getStorages()) { XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } dt.getStorageTables().getStorageTable().add(tblElement); } } return dt; } /**
public XStorageTableElement getStorageOfFact(LensSessionHandle sessionid, String fact, String storageName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient msClient = getClient(sessionid); FactTable factTable = msClient.getFactTable(fact); Set<UpdatePeriod> updatePeriods = factTable.getUpdatePeriods().get(storageName); XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName))); tblElement.setStorageName(storageName); for (UpdatePeriod p : updatePeriods) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } return tblElement; } }
public XStorageTableElement getStorageOfDim(LensSessionHandle sessionid, String dimTblName, String storageName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); CubeDimensionTable dimTable = msClient.getDimensionTable(dimTblName); XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } return tblElement; } }
private void validateDim(Dimension udim, Set<CubeDimAttribute> attrs, String basedim, String referdim) throws HiveException, LensException { assertTrue(client.tableExists(udim.getName())); Table dimTbl = client.getHiveTable(udim.getName()); assertTrue(client.isDimension(dimTbl)); Dimension dim = new Dimension(dimTbl); assertTrue(udim.equals(dim), "Equals failed for " + dim.getName()); assertTrue(udim.equals(client.getDimension(udim.getName()))); assertEquals(dim.getAttributes().size(), attrs.size()); assertNotNull(dim.getAttributeByName(basedim)); assertTrue(dim.getAttributeByName(basedim) instanceof BaseDimAttribute); if (referdim != null) { assertNotNull(dim.getAttributeByName(referdim)); assertTrue(dim.getAttributeByName(referdim) instanceof ReferencedDimAttribute); } assertEquals(udim.getAttributeNames().size() + udim.getExpressionNames().size(), dim.getAllFieldNames().size()); }
getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName))); tblElement.setStorageName(storageName); for (UpdatePeriod p : updatePeriods) { XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor(); updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), (String) entry.getValue())))); updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod) entry.getKey()).name()));
final Table tbl = CubeMetastoreClient.getInstance(context.getHiveConf()).getHiveTable(lensTable); final String index = tbl.getProperty(LensConfConstants.ES_INDEX_NAME); final String type = tbl.getProperty(LensConfConstants.ES_TYPE_NAME);
final Table tbl = CubeMetastoreClient.getInstance(context.getHiveConf()).getHiveTable(lensTable); final String index = tbl.getProperty(LensConfConstants.ES_INDEX_NAME); final String type = tbl.getProperty(LensConfConstants.ES_TYPE_NAME);
Table cubeTbl = client.getHiveTable(dimName); assertTrue(client.isDimensionTable(cubeTbl)); String storageTableName = getFactOrDimtableStorageTableName(dimName, storageName); assertTrue(client.tableExists(storageTableName)); assertTrue(!client.getHiveTable(storageTableName).isPartitioned());
Table cubeTbl = client.getHiveTable(cubename); assertTrue(client.isCube(cubeTbl)); Cube cube2 = new Cube(cubeTbl);
Table cubeTbl = client.getHiveTable(dimName); assertTrue(client.isDimensionTable(cubeTbl)); String storageTableName2 = getFactOrDimtableStorageTableName(dimName, c2); assertTrue(client.tableExists(storageTableName2)); assertTrue(!client.getHiveTable(storageTableName2).isPartitioned());
@Test(priority = 2) public void testAlterDerivedCube() throws Exception { String name = "alter_derived_cube"; client.createDerivedCube(CUBE_NAME, name, measures, dimensions, emptyHashMap, 0L); // Test alter cube Table cubeTbl = client.getHiveTable(name); DerivedCube toAlter = new DerivedCube(cubeTbl, (Cube) client.getCube(CUBE_NAME)); toAlter.addMeasure("msr4"); toAlter.removeMeasure("msr3"); toAlter.addDimension("dim1StartTime"); toAlter.removeDimension("dim1"); assertNotNull(toAlter.getMeasureByName("msr4")); assertNotNull(toAlter.getMeasureByName("msr2")); assertNull(toAlter.getMeasureByName("msr3")); assertNotNull(toAlter.getDimAttributeByName("dim1StartTime")); assertNotNull(toAlter.getDimAttributeByName("dim2")); assertNull(toAlter.getDimAttributeByName("dim1")); client.alterCube(name, toAlter); DerivedCube altered = (DerivedCube) client.getCube(name); assertEquals(toAlter, altered); assertNotNull(altered.getMeasureByName("msr4")); CubeMeasure addedMsr = altered.getMeasureByName("msr4"); assertEquals(addedMsr.getType(), "bigint"); assertNotNull(altered.getDimAttributeByName("dim1StartTime")); BaseDimAttribute addedDim = (BaseDimAttribute) altered.getDimAttributeByName("dim1StartTime"); assertEquals(addedDim.getType(), "string"); assertNotNull(addedDim.getStartTime()); client.dropCube(name); assertFalse(client.tableExists(name)); }
Table cubeTbl = client.getHiveTable(factName); assertTrue(client.isFactTable(cubeTbl)); assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME));
Table cubeTbl = client.getHiveTable(factNameWithPart); assertTrue(client.isFactTable(cubeTbl)); assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME));
String segmentName = "testMetastoreSegmentation"; Table cubeTbl = client.getHiveTable(CUBE_NAME); assertTrue(client.isCube(cubeTbl));