@Override public int addPartitionToDimStorage(LensSessionHandle sessionid, String dimTblName, String storageName, XPartition partition) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ checkDimTableStorage(sessionid, dimTblName, storageName); return getClient(sessionid).addPartition(storagePartSpecFromXPartition(partition), storageName, CubeTableType.DIM_TABLE).size(); } catch (HiveException exc) { throw new LensException(exc); } }
StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); try { client.addPartition(sPartSpec, c1, CubeTableType.FACT); } catch (HiveException e) { log.error("Encountered Hive exception.", e); timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c1, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); timeParts.put("ttd2", temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c4, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime();
final StoragePartitionDesc partSpec6 = new StoragePartitionDesc(cubeDim.getName(), timeParts6, null, HOURLY); client.addPartition(partSpec6, c1, CubeTableType.DIM_TABLE); final StoragePartitionDesc partSpec7 = new StoragePartitionDesc(cubeDim.getName(), timeParts7, null, HOURLY); client.addPartition(partSpec7, c1, CubeTableType.DIM_TABLE); assertEquals(client.getAllParts(c1TableName).size(), 12); assertEquals(getLatestValues(c1TableName, HOURLY, partColNames, null), toPartitionArray(HOURLY, 1, 1, 1));
@Override public int addPartitionToFactStorage(LensSessionHandle sessionid, String fact, String storageName, XPartition partition) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { checkFactStorage(sessionid, fact, storageName); return getClient(sessionid) .addPartition(storagePartSpecFromXPartition(partition), storageName, CubeTableType.FACT).size(); } catch (HiveException exc) { throw new LensException(exc); } }
final StoragePartitionDesc partSpec6 = new StoragePartitionDesc(cubeFact.getName(), timeParts6, null, HOURLY); client.addPartition(partSpec6, c1, CubeTableType.FACT); client.addPartition(partSpec6, c2, CubeTableType.FACT); final StoragePartitionDesc partSpec7 = new StoragePartitionDesc(cubeFact.getName(), timeParts7, null, HOURLY); client.addPartition(partSpec7, c1, CubeTableType.FACT); client.addPartition(partSpec7, c2, CubeTableType.FACT);
private void createBaseCubeFactPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "testFact5_RAW_BASE"; CubeFactTable fact = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); Date temp = cal.getTime(); while (!(temp.after(NOW))) { Map<String, Date> timeParts = new HashMap<String, Date>(); timeParts.put("dt", temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c1, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } }
private void createTestFact2RawPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "testFact2_raw"; CubeFactTable fact2 = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); Date temp = cal.getTime(); while (!(temp.after(NOW))) { Map<String, Date> timeParts = new HashMap<String, Date>(); timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact2.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c3, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } }
private void createPIParts(CubeMetastoreClient client, CubeFactTable fact, String storageName) throws Exception { // Add partitions in PI storage //daily partition registered for pt=day1, it = day1 //hourly partitions registered for pt=day1-hours[0-23] it=day1-hours[0-23] Calendar pcal = Calendar.getInstance(); pcal.setTime(ONEDAY_BACK); pcal.set(HOUR_OF_DAY, 0); Calendar ical = Calendar.getInstance(); ical.setTime(ONEDAY_BACK); ical.set(HOUR_OF_DAY, 0); Date ptime = pcal.getTime(); Date itime = ical.getTime(); Map<String, Date> timeParts = new HashMap<String, Date>(); timeParts.put("pt", ptime); timeParts.put("it", itime); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, DAILY); client.addPartition(sPartSpec, storageName, CubeTableType.FACT); for (int i = 0; i < 24; i++) { ptime = pcal.getTime(); itime = ical.getTime(); timeParts.put("pt", ptime); timeParts.put("it", itime); sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, storageName, CubeTableType.FACT); pcal.add(HOUR_OF_DAY, 1); ical.add(HOUR_OF_DAY, 1); } }
private void createCubeCheapFactPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "cheapFact"; CubeFactTable fact = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); Date temp = cal.getTime(); while (!(temp.after(NOW))) { Map<String, Date> timeParts = new HashMap<String, Date>(); timeParts.put("ttd", temp); timeParts.put("ttd2", temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c99, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } // Add all hourly partitions for TWO_DAYS_RANGE_BEFORE_4_DAYS cal.setTime(BEFORE_6_DAYS); temp = cal.getTime(); while (!(temp.after(BEFORE_4_DAYS))) { Map<String, Date> timeParts = new HashMap<String, Date>(); timeParts.put("ttd", temp); timeParts.put("ttd2", temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c99, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } }
pTimes.get(DAILY).add(ptime); iTimes.get(DAILY).add(itime); client.addPartition(sPartSpec, storageName, CubeTableType.FACT); pcal.add(DAY_OF_MONTH, 1); ical.add(HOUR_OF_DAY, 20); pTimes.get(DAILY).add(ptime); iTimes.get(DAILY).add(itime); client.addPartition(sPartSpec, storageName, CubeTableType.FACT); pTimes.get(HOURLY).add(ptime); iTimes.get(HOURLY).add(itime); client.addPartition(sPartSpec, storageName, CubeTableType.FACT); pcal.add(HOUR_OF_DAY, 1); ical.add(HOUR_OF_DAY, 1); pTimes.get(DAILY).add(ptime); iTimes.get(DAILY).add(itime); client.addPartition(sPartSpec, storageName, CubeTableType.FACT); } else if (p == 3) { // day3 pTimes.get(HOURLY).add(ptime); iTimes.get(HOURLY).add(itime); client.addPartition(sPartSpec, storageName, CubeTableType.FACT);
nonTimeParts.put("region", "asia"); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, HOURLY); client.addPartition(sPartSpec, c3, CubeTableType.DIM_TABLE); expectedLatestValues.put("asia", NOW); assertLatestForRegions(storageTableName, expectedLatestValues); nonTimeParts.put("region", "africa"); sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, HOURLY); client.addPartition(sPartSpec, c3, CubeTableType.DIM_TABLE); expectedLatestValues.put("asia", NOW); expectedLatestValues.put("africa", getDateWithOffset(HOURLY, -1)); nonTimeParts.put("region", "africa"); sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, HOURLY); client.addPartition(sPartSpec, c3, CubeTableType.DIM_TABLE); expectedLatestValues.put("asia", NOW); expectedLatestValues.put("africa", getDateWithOffset(HOURLY, 1)); nonTimeParts.put("region", "asia"); sPartSpec = new StoragePartitionDesc(dimName, timeParts, nonTimeParts, HOURLY); client.addPartition(sPartSpec, c3, CubeTableType.DIM_TABLE); expectedLatestValues.put("asia", getDateWithOffset(HOURLY, 3)); expectedLatestValues.put("africa", getDateWithOffset(HOURLY, 1));
client.addPartition(partSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey()));
client.addPartition(sPartSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timeParts, partSpec)); assertTrue(client.latestPartitionExists(cubeFactWithParts.getName(), c1, getDatePartitionKey()));
client.addPartition(sPartSpec, c1, CubeTableType.DIM_TABLE); assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timeParts)); assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, getDatePartitionKey()));
client.addPartition(partSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, testDtPart.getName()));
client.addPartition(sPartSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithParts.getName(), c1, HOURLY, timeParts, partSpec)); assertTrue(client.latestPartitionExists(cubeFactWithParts.getName(), c1, getDatePartitionKey()));
client.addPartition(partSpec, c1, CubeTableType.FACT); fail("Add should fail since non_existing_part_col is non-existing"); } catch (LensException e) { client.addPartition(partSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); partSpec2.setInputFormat(SequenceFileInputFormat.class.getCanonicalName()); partSpec2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); client.addPartition(partSpec2, c1, CubeTableType.FACT); assertEquals(client.getAllParts(storageTableName).size(), 1); assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap));
client.addPartition(sPartSpec, c1, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithTwoStorages.getName(), c1, HOURLY, timeParts, partSpec)); assertTrue(client.latestPartitionExists(cubeFactWithTwoStorages.getName(), c1, getDatePartitionKey())); client.addPartition(sPartSpec2, c2, CubeTableType.FACT); assertTrue(client.factPartitionExists(cubeFactWithTwoStorages.getName(), c2, HOURLY, timeParts, emptyHashMap)); assertTrue(client.latestPartitionExists(cubeFactWithTwoStorages.getName(), c2, getDatePartitionKey()));
StoragePartitionDesc sPartSpec0 = new StoragePartitionDesc(cubeDim.getName(), timePartsNow, null, HOURLY); client.addPartition(sPartSpec0, c1, CubeTableType.DIM_TABLE); assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timePartsNow)); assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, getDatePartitionKey())); sPartSpec1.setInputFormat(SequenceFileInputFormat.class.getCanonicalName()); sPartSpec1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); client.addPartition(sPartSpec1, c1, CubeTableType.DIM_TABLE); client.addPartition(sPartSpec2, c1, CubeTableType.DIM_TABLE); assertFalse(client.dimTableLatestPartitionExists(storageTableName)); client.addPartition(sPartSpec1, c1, CubeTableType.DIM_TABLE); assertTrue(client.dimTableLatestPartitionExists(storageTableName)); client.dropStorageFromDim(cubeDim.getName(), c1);