public static InputJobInfo getLastInputJobInfosFromConf(Configuration conf) throws IOException { LinkedList<InputJobInfo> inputJobInfos = getInputJobInfosFromConf(conf); if (inputJobInfos == null || inputJobInfos.isEmpty()) { return null; } else { return getInputJobInfosFromConf(conf).getLast(); } }
/** * Get statistics about the data to be loaded. Only input data size is implemented at this time. */ @Override public ResourceStatistics getStatistics(String location, Job job) throws IOException { try { if (dbName == null || tableName == null) { throw new IOException("DB or table name unset. setLocation() must be invoked on this " + "loader to set them"); } ResourceStatistics stats = new ResourceStatistics(); long inputSize = -1; LinkedList<InputJobInfo> inputJobInfos = HCatUtil.getInputJobInfosFromConf( job.getConfiguration()); for (InputJobInfo inputJobInfo : inputJobInfos) { if (dbName.equals(inputJobInfo.getDatabaseName()) && tableName.equals(inputJobInfo.getTableName())){ inputSize = getSizeInBytes(inputJobInfo); break; } } if (inputSize == -1) { throw new IOException("Could not calculate input size for database: " + dbName + ", " + "table: " + tableName + ". Requested location:" + location); } stats.setSizeInBytes(inputSize); return stats; } catch (Exception e) { throw new IOException(e); } }
@Test public void testInputJobInfoInConf() throws Exception { Configuration conf = new Configuration(false); InputJobInfo inputJobInfo = HCatUtil.getLastInputJobInfosFromConf(conf); Assert.assertNull(inputJobInfo); List<InputJobInfo> inputJobInfos = HCatUtil.getInputJobInfosFromConf(conf); Assert.assertNull(inputJobInfos); InputJobInfo inputJobInfo0 = InputJobInfo.create("db", "table", "", new Properties()); InputJobInfo inputJobInfo1 = InputJobInfo.create("db", "table2", "", new Properties()); HCatUtil.putInputJobInfoToConf(inputJobInfo0, conf); HCatUtil.putInputJobInfoToConf(inputJobInfo1, conf); inputJobInfo = HCatUtil.getLastInputJobInfosFromConf(conf); inputJobInfos = HCatUtil.getInputJobInfosFromConf(conf); Assert.assertEquals(inputJobInfo1.getDatabaseName(), inputJobInfo.getDatabaseName()); Assert.assertEquals(inputJobInfo1.getTableName(), inputJobInfo.getTableName()); Assert.assertEquals(inputJobInfo0.getDatabaseName(), inputJobInfos.get(0).getDatabaseName()); Assert.assertEquals(inputJobInfo0.getTableName(), inputJobInfos.get(0).getTableName()); } }