SnapshotTable snapSrc = srcSnapMgr.getSnapshotTable(item); long ts = snapSrc.getLastModified(); snapSrc.setLastModified(0); SnapshotTable snapSaved = dstSnapMgr.trySaveNewSnapshot(snapSrc); snapSrc.setLastModified(ts); for (Map.Entry<String, String> entry : segment.getSnapshots().entrySet()) { if (entry.getValue().equalsIgnoreCase(item)) { entry.setValue(snapSaved.getResourcePath()); logger.info("Item " + item + " is dup, instead " + snapSaved.getResourcePath() + " is reused");
response.setSnapshotID(metaStoreTableSnapshot.getId()); response.setSnapshotType(TableSnapshotResponse.TYPE_INNER); response.setLastBuildTime(metaStoreTableSnapshot.getLastBuildTime()); response.setStorageType(SnapshotTable.STORAGE_TYPE_METASTORE); response.setSourceTableSize(metaStoreTableSnapshot.getSignature().getSize()); response.setSourceTableLastModifyTime(metaStoreTableSnapshot.getSignature().getLastModifiedTime()); response.setCubesAndSegmentsUsage(snapshotUsageMap.get(metaStoreTableSnapshot.getResourcePath())); result.add(response);
public SnapshotTable rebuildSnapshot(IReadableTable table, TableDesc tableDesc, String overwriteUUID) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.setUuid(overwriteUUID); snapshot.takeSnapshot(table, tableDesc); SnapshotTable existing = getSnapshotTable(snapshot.getResourcePath()); snapshot.setLastModified(existing.getLastModified()); save(snapshot); snapshotCache.put(snapshot.getResourcePath(), snapshot); return snapshot; }
private String checkDupByInfo(SnapshotTable snapshot) throws IOException { ResourceStore store = getStore(); String resourceDir = snapshot.getResourceDir(); NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; TableSignature sig = snapshot.getSignature(); for (String existing : existings) { SnapshotTable existingTable = load(existing, false); // skip cache, // direct load from store if (existingTable != null && sig.equals(existingTable.getSignature())) return existing; } return null; }
private String checkDupByContent(SnapshotTable snapshot) throws IOException { ResourceStore store = getStore(); String resourceDir = snapshot.getResourceDir(); NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; for (String existing : existings) { SnapshotTable existingTable = load(existing, true); // skip cache, direct load from store if (existingTable != null && existingTable.equals(snapshot)) return existing; } return null; }
private void save(SnapshotTable snapshot) throws IOException { ResourceStore store = getStore(); String path = snapshot.getResourcePath(); store.putBigResource(path, snapshot, System.currentTimeMillis(), SnapshotTableSerializer.FULL_SERIALIZER); }
public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
@Test public void basicTest() throws Exception { String tableName = "EDW.TEST_SITES"; HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName); TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName); String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath(); snapshotMgr.wipeoutCache(); SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath); // compare hive & snapshot TableReader hiveReader = hiveTable.getReader(); TableReader snapshotReader = snapshot.getReader(); while (true) { boolean hiveNext = hiveReader.next(); boolean snapshotNext = snapshotReader.next(); assertEquals(hiveNext, snapshotNext); if (hiveNext == false) break; String[] hiveRow = hiveReader.getRow(); String[] snapshotRow = snapshotReader.getRow(); assertArrayEquals(hiveRow, snapshotRow); } } }
public SnapshotTable buildSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException { SnapshotTable snapshot = new SnapshotTable(table); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
public SnapshotTable trySaveNewSnapshot(SnapshotTable snapshotTable) throws IOException { String dupTable = checkDupByContent(snapshotTable); if (dupTable != null) { logger.info("Identical snapshot content " + snapshotTable + ", reuse existing snapshot at " + dupTable); return getSnapshotTable(dupTable); } save(snapshotTable); snapshotCache.put(snapshotTable.getResourcePath(), snapshotTable); return snapshotTable; }
public SnapshotTable buildSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException { SnapshotTable snapshot = new SnapshotTable(table); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if (snapshot.getSignature().getSize() / 1024 / 1024 > config.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + config.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
private String checkDupByInfo(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); ArrayList<String> existings = store.listResources(resourceDir); if (existings == null) return null; TableSignature sig = snapshot.getSignature(); for (String existing : existings) { SnapshotTable existingTable = load(existing, false); // skip cache, // direct // load from // store if (existingTable != null && sig.equals(existingTable.getSignature())) return existing; } return null; }
private String checkDupByContent(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); ArrayList<String> existings = store.listResources(resourceDir); if (existings == null) return null; for (String existing : existings) { SnapshotTable existingTable = load(existing, true); // skip cache, direct load from store if (existingTable != null && existingTable.equals(snapshot)) return existing; } return null; }
SnapshotTable snapSrc = srcSnapMgr.getSnapshotTable(item); long ts = snapSrc.getLastModified(); snapSrc.setLastModified(0); SnapshotTable snapSaved = dstSnapMgr.trySaveNewSnapshot(snapSrc); snapSrc.setLastModified(ts); for (Map.Entry<String, String> entry : segment.getSnapshots().entrySet()) { if (entry.getValue().equalsIgnoreCase(item)) { entry.setValue(snapSaved.getResourcePath()); logger.info("Item " + item + " is dup, instead " + snapSaved.getResourcePath() + " is reused");
private static void rebuild(String table, String overwriteUUID, String project) throws IOException { KylinConfig conf = KylinConfig.getInstanceFromEnv(); TableMetadataManager metaMgr = TableMetadataManager.getInstance(conf); SnapshotManager snapshotMgr = SnapshotManager.getInstance(conf); TableDesc tableDesc = metaMgr.getTableDesc(table, project); if (tableDesc == null) throw new IllegalArgumentException("Not table found by " + table); if (tableDesc.isView()) throw new IllegalArgumentException("Build snapshot of hive view \'" + table + "\' not supported."); SnapshotTable snapshot = snapshotMgr.rebuildSnapshot(SourceManager.createReadableTable(tableDesc, null), tableDesc, overwriteUUID); System.out.println("resource path updated: " + snapshot.getResourcePath()); } }
public SnapshotTable rebuildSnapshot(IReadableTable table, TableDesc tableDesc, String overwriteUUID) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.setUuid(overwriteUUID); snapshot.takeSnapshot(table, tableDesc); SnapshotTable existing = getSnapshotTable(snapshot.getResourcePath()); snapshot.setLastModified(existing.getLastModified()); save(snapshot); snapshotCache.put(snapshot.getResourcePath(), snapshot); return snapshot; }
public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.updateRandomUuid(); String dup = checkDupByInfo(snapshot); if (dup != null) { logger.info("Identical input " + table.getSignature() + ", reuse existing snapshot at " + dup); return getSnapshotTable(dup); } if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) { throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() // + " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize()); } snapshot.takeSnapshot(table, tableDesc); return trySaveNewSnapshot(snapshot); }
private String checkDupByInfo(SnapshotTable snapshot) throws IOException { ResourceStore store = getStore(); String resourceDir = snapshot.getResourceDir(); NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; TableSignature sig = snapshot.getSignature(); for (String existing : existings) { SnapshotTable existingTable = load(existing, false); // skip cache, // direct load from store if (existingTable != null && sig.equals(existingTable.getSignature())) return existing; } return null; }
private String checkDupByContent(SnapshotTable snapshot) throws IOException { ResourceStore store = getStore(); String resourceDir = snapshot.getResourceDir(); NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; for (String existing : existings) { SnapshotTable existingTable = load(existing, true); // skip cache, direct load from store if (existingTable != null && existingTable.equals(snapshot)) return existing; } return null; }
SnapshotTable snapSrc = srcSnapMgr.getSnapshotTable(item); long ts = snapSrc.getLastModified(); snapSrc.setLastModified(0); SnapshotTable snapSaved = dstSnapMgr.trySaveNewSnapshot(snapSrc); snapSrc.setLastModified(ts); for (Map.Entry<String, String> entry : segment.getSnapshots().entrySet()) { if (entry.getValue().equalsIgnoreCase(item)) { entry.setValue(snapSaved.getResourcePath()); logger.info("Item " + item + " is dup, instead " + snapSaved.getResourcePath() + " is reused");