private FileTable getFileTable() throws IOException { if (fileTable == null) { fileTable = new FileTable(getHDFSLocation(true), nColumns); } return fileTable; }
@Override public String getColumnDelimeter() throws IOException { return getFileTable().getColumnDelimeter(); }
@Override public TableSignature getSignature() throws IOException { return getFileTable().getSignature(); }
@Override public TableSignature getSignature() throws IOException { if (!exists()) throw new IllegalStateException("Table not exists :" + path); try { Pair<Long, Long> sizeAndLastModified = getSizeAndLastModified(path); return new TableSignature(path, sizeAndLastModified.getFirst(), sizeAndLastModified.getSecond()); } catch (FileNotFoundException ex) { return null; } }
@Override public TableSignature getSignature() throws IOException { try { String path = computeHDFSLocation(); Pair<Long, Long> sizeAndLastModified = FileTable.getSizeAndLastModified(path); long size = sizeAndLastModified.getFirst(); long lastModified = sizeAndLastModified.getSecond(); // for non-native hive table, cannot rely on size & last modified on HDFS if (getHiveClient().isNativeTable(database, hiveTable) == false) { lastModified = System.currentTimeMillis(); // assume table is ever changing } return new TableSignature(path, size, lastModified); } catch (Exception e) { if (e instanceof IOException) throw (IOException) e; else throw new IOException(e); } }
@Test public void testBasic() throws Exception { TableDesc siteTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("EDW.TEST_SITES"); TableDesc categoryTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.test_category_groupings"); LookupBytesTable lookup; System.out.println("============================================================================"); File f = new File(LOCALMETA_TEST_DATA + "/data/EDW.TEST_SITES.csv"); lookup = new LookupBytesTable(siteTable, new String[] { "SITE_ID" }, new FileTable("file://" + f.getAbsolutePath(), 10)); lookup.dump(); System.out.println("============================================================================"); f = new File(LOCALMETA_TEST_DATA + "/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv"); lookup = new LookupBytesTable(categoryTable, new String[] { "leaf_categ_id", "site_id" }, new FileTable("file://" + f.getAbsolutePath(), 36)); lookup.dump(); System.out.println("============================================================================"); ByteArray k1 = new ByteArray(Bytes.toBytes("533")); ByteArray k2 = new ByteArray(Bytes.toBytes("0")); Array<ByteArray> key = new Array<ByteArray>(new ByteArray[] { k1, k2 }); System.out.println(lookup.getRow(key)); } }
srcColIdx = col.getColumn().getZeroBasedIndex(); int nColumns = metaMgr.getTableDesc(col.getTable()).getColumnCount(); table = new FileTable(factColumnsPath + "/" + col.getName(), nColumns); return new Object[] { srcTable, srcCol, srcColIdx, table }; srcColIdx = col.getColumn().getZeroBasedIndex(); if (model.isFactTable(col.getTable())) { table = new FileTable(factColumnsPath + "/" + col.getName(), -1); } else { table = new HiveTable(metaMgr, col.getTable()); srcCol = dict; srcColIdx = 0; table = new FileTable(dictDataSetPath, -1);
srcColIdx = col.getColumn().getZeroBasedIndex(); int nColumns = metaMgr.getTableDesc(col.getTable()).getColumnCount(); table = new FileTable(factColumnsPath + "/" + col.getName(), nColumns); return new Object[] { srcTable, srcCol, srcColIdx, table }; srcColIdx = col.getColumn().getZeroBasedIndex(); if (model.isFactTable(col.getTable())) { table = new FileTable(factColumnsPath + "/" + col.getName(), -1); } else { table = new HiveTable(metaMgr, col.getTable()); srcCol = dict; srcColIdx = 0; table = new FileTable(dictDataSetPath, -1);