@Override public String getName() { return name(); }
@Override public String getName() { return name(); }
public static String getLookAheadPTPartsKey(UpdatePeriod interval) { return LOOK_AHEAD_PT_PARTS_PFX + interval.name().toLowerCase(); }
public static String getLookAheadPTPartsKey(UpdatePeriod interval) { return LOOK_AHEAD_PT_PARTS_PFX + interval.name().toLowerCase(); }
public static String getLookAheadTimePartsKey(UpdatePeriod interval) { return LOOK_AHEAD_TIME_PARTS_PFX + interval.name().toLowerCase(); }
private static void setSnapshotDumpPeriods(String name, Map<String, String> props, Map<String, UpdatePeriod> snapshotDumpPeriods) { if (snapshotDumpPeriods != null) { props.put(MetastoreUtil.getDimensionStorageListKey(name), MetastoreUtil.getStr(snapshotDumpPeriods.keySet())); for (Map.Entry<String, UpdatePeriod> entry : snapshotDumpPeriods.entrySet()) { if (entry.getValue() != null) { props.put(MetastoreUtil.getDimensionDumpPeriodKey(name, entry.getKey()), entry.getValue().name()); } } } }
private static void setSnapshotDumpPeriods(String name, Map<String, String> props, Map<String, UpdatePeriod> snapshotDumpPeriods) { if (snapshotDumpPeriods != null) { props.put(MetastoreUtil.getDimensionStorageListKey(name), MetastoreUtil.getStr(snapshotDumpPeriods.keySet())); for (Map.Entry<String, UpdatePeriod> entry : snapshotDumpPeriods.entrySet()) { if (entry.getValue() != null) { props.put(MetastoreUtil.getDimensionDumpPeriodKey(name, entry.getKey()), entry.getValue().name()); } } } }
/** * Normalized cost of an update period. update period weight multiplied by number of partitions of that update period. * * @param updatePeriod * @param driver * @return normalized cost. * @throws LensException */ private double getNormalizedUpdatePeriodCost(final UpdatePeriod updatePeriod, LensDriver driver) throws LensException { double weight = driver.getConf().getDouble(UPDATE_PERIOD_WEIGHT_PREFIX + updatePeriod.name().toLowerCase(), updatePeriod.getNormalizationFactor()); return weight * updatePeriod.weight() / UpdatePeriod.DAILY.weight(); }
/** * Normalized cost of an update period. update period weight multiplied by number of partitions of that update period. * * @param updatePeriod * @param driver * @return normalized cost. * @throws LensException */ private double getNormalizedUpdatePeriodCost(final UpdatePeriod updatePeriod, LensDriver driver) throws LensException { double weight = driver.getConf().getDouble(UPDATE_PERIOD_WEIGHT_PREFIX + updatePeriod.name().toLowerCase(), updatePeriod.getNormalizationFactor()); return weight * updatePeriod.weight() / UpdatePeriod.DAILY.weight(); }
private void assertTimeline(CubeMetastoreClient client, String factName, String storageName, UpdatePeriod updatePeriod, String timeDim, PartitionTimeline expectedTimeline) throws Exception { assertNotNull(factName); assertNotNull(storageName); assertNotNull(updatePeriod); assertNotNull(timeDim); String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName); List<PartitionTimeline> timelines = client.getTimelines(factName, storageName, updatePeriod.name(), timeDim); assertEquals(timelines.size(), 1); PartitionTimeline actualTimeline = timelines.get(0); assertEquals(actualTimeline, expectedTimeline); assertEquals(client.getTable(storageTableName).getParameters() .get(MetastoreUtil.getPartitionTimelineStorageClassKey(updatePeriod, timeDim)), expectedTimeline.getClass().getCanonicalName()); expectedTimeline.init(client.getTable(MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName))); assertEquals(actualTimeline, expectedTimeline); }
public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); if (!dimTable.getStorages().isEmpty()) { for (String storageName : dimTable.getStorages()) { XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } dt.getStorageTables().getStorageTable().add(tblElement); } } return dt; }
public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); if (!dimTable.getStorages().isEmpty()) { for (String storageName : dimTable.getStorages()) { XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } dt.getStorageTables().getStorageTable().add(tblElement); } } return dt; } /**
public XStorageTableElement getStorageOfFact(LensSessionHandle sessionid, String fact, String storageName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient msClient = getClient(sessionid); FactTable factTable = msClient.getFactTable(fact); Set<UpdatePeriod> updatePeriods = factTable.getUpdatePeriods().get(storageName); XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName))); tblElement.setStorageName(storageName); for (UpdatePeriod p : updatePeriods) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } return tblElement; } }
public XStorageTableElement getStorageOfDim(LensSessionHandle sessionid, String dimTblName, String storageName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); CubeDimensionTable dimTable = msClient.getDimensionTable(dimTblName); XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } return tblElement; } }
tblElement.setStorageName(storageName); for (UpdatePeriod p : updatePeriods) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), (String) entry.getValue())))); updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod) entry.getKey()).name())); xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor);
tblElement.setStorageName(storageName); for (UpdatePeriod p : updatePeriods) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), (String) entry.getValue())))); updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod) entry.getKey()).name())); xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor);
updatePeriod, sc.getStorageTable(), maxInterval); skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.UPDATE_PERIOD_BIGGER_THAN_MAX); } else if (validUpdatePeriods != null && !validUpdatePeriods.contains(updatePeriod.name().toLowerCase())) {
updatePeriod, sc.getStorageTable(), maxInterval); skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.UPDATE_PERIOD_BIGGER_THAN_MAX); } else if (validUpdatePeriods != null && !validUpdatePeriods.contains(updatePeriod.name().toLowerCase())) {
partParams = new HashMap<String, String>(); partParams.put(MetastoreConstants.PARTITION_UPDATE_PERIOD, addPartitionDesc.getUpdatePeriod().name()); addParts.addPartition(addPartitionDesc.getStoragePartSpec(), location); int curIndex = addParts.getPartitionCount() - 1;
partParams = new HashMap<String, String>(); partParams.put(MetastoreConstants.PARTITION_UPDATE_PERIOD, addPartitionDesc.getUpdatePeriod().name()); addParts.addPartition(addPartitionDesc.getStoragePartSpec(), location); int curIndex = addParts.getPartitionCount() - 1;