@Test public void testTableInputFormatBaseReverseDNSForIPv6() throws UnknownHostException { String address = "ipv6.google.com"; String localhost = null; InetAddress addr = null; TableInputFormat inputFormat = new TableInputFormat(); try { localhost = InetAddress.getByName(address).getCanonicalHostName(); addr = Inet6Address.getByName(address); } catch (UnknownHostException e) { // google.com is down, we can probably forgive this test. return; } System.out.println("Should retrun the hostname for this host " + localhost + " addr : " + addr); String actualHostName = inputFormat.reverseDNS(addr); assertEquals("Should retrun the hostname for this host. Expected : " + localhost + " Actual : " + actualHostName, localhost, actualHostName); }
/** * Tests Number of inputSplits for MR job when specify number of mappers for TableInputFormatXXX * This test does not run MR job */ protected void testNumOfSplits(int splitsPerRegion, int expectedNumOfSplits) throws IOException, InterruptedException, ClassNotFoundException { String jobName = "TestJobForNumOfSplits"; LOG.info("Before map/reduce startup - job " + jobName); Configuration c = new Configuration(TEST_UTIL.getConfiguration()); Scan scan = new Scan(); scan.addFamily(INPUT_FAMILYS[0]); scan.addFamily(INPUT_FAMILYS[1]); c.setInt("hbase.mapreduce.tableinput.mappers.per.region", splitsPerRegion); c.set(KEY_STARTROW, ""); c.set(KEY_LASTROW, ""); Job job = Job.getInstance(c, jobName); TableMapReduceUtil.initTableMapperJob(TABLE_NAME.getNameAsString(), scan, ScanMapper.class, ImmutableBytesWritable.class, ImmutableBytesWritable.class, job); TableInputFormat tif = new TableInputFormat(); tif.setConf(job.getConfiguration()); Assert.assertEquals(TABLE_NAME, table.getName()); List<InputSplit> splits = tif.getSplits(job); Assert.assertEquals(expectedNumOfSplits, splits.size()); }
/** * Run MR job to test autobalance for setting number of mappers for TIF This does not run real MR * job */ protected void testAutobalanceNumOfSplit() throws IOException { // set up splits for testing List<InputSplit> splits = new ArrayList<>(5); int[] regionLen = { 10, 20, 20, 40, 60 }; for (int i = 0; i < 5; i++) { InputSplit split = new TableSplit(TABLE_NAME, new Scan(), Bytes.toBytes(i), Bytes.toBytes(i + 1), "", "", regionLen[i] * 1048576); splits.add(split); } TableInputFormat tif = new TableInputFormat(); List<InputSplit> res = tif.calculateAutoBalancedSplits(splits, 1073741824); assertEquals("Saw the wrong number of splits", 5, res.size()); TableSplit ts1 = (TableSplit) res.get(0); assertEquals("The first split end key should be", 2, Bytes.toInt(ts1.getEndRow())); TableSplit ts2 = (TableSplit) res.get(1); assertEquals("The second split regionsize should be", 20 * 1048576, ts2.getLength()); TableSplit ts3 = (TableSplit) res.get(2); assertEquals("The third split start key should be", 3, Bytes.toInt(ts3.getStartRow())); TableSplit ts4 = (TableSplit) res.get(4); assertNotEquals("The seventh split start key should not be", 4, Bytes.toInt(ts4.getStartRow())); } }
public HBaseInputFormat() { inputFormat = new TableInputFormat(); }
private TableInputFormat getDelegate(Configuration conf) throws IOException { TableInputFormat delegate = new TableInputFormat(); String tableName = HBaseMetadataProvider.getTableName(dataset.getName()); conf.set(TableInputFormat.INPUT_TABLE, tableName); if (view != null) { Job tempJob = new Job(); Scan scan = ((BaseEntityScanner) view.newEntityScanner()).getScan(); TableMapReduceUtil.initTableMapperJob(tableName, scan, TableMapper.class, null, null, tempJob); Configuration tempConf = Hadoop.JobContext.getConfiguration.invoke(tempJob); conf.set(SCAN, tempConf.get(SCAN)); } delegate.setConf(conf); return delegate; }
private TableInputFormat getDelegate(Configuration conf) throws IOException { TableInputFormat delegate = new TableInputFormat(); String tableName = HBaseMetadataProvider.getTableName(dataset.getName()); conf.set(TableInputFormat.INPUT_TABLE, tableName); if (view != null) { Job tempJob = new Job(); Scan scan = ((BaseEntityScanner) view.newEntityScanner()).getScan(); TableMapReduceUtil.initTableMapperJob(tableName, scan, TableMapper.class, null, null, tempJob); Configuration tempConf = Hadoop.JobContext.getConfiguration.invoke(tempJob); conf.set(SCAN, tempConf.get(SCAN)); } delegate.setConf(conf); return delegate; }
private static TableInputFormat getNewTableInputFormat(Connection connection, TableName tableName) throws IOException, NoSuchFieldException, IllegalAccessException { TableInputFormat tableInputFormat = new TableInputFormat(); HbaseClient.inject(TableInputFormatBase.class, tableInputFormat, "table", connection.getTable(tableName)); HbaseClient.inject(TableInputFormatBase.class, tableInputFormat, "regionLocator", connection.getRegionLocator(tableName)); HbaseClient.inject(TableInputFormatBase.class, tableInputFormat, "admin", connection.getAdmin()); return tableInputFormat; }
@Test public void testTableInputFormatBaseReverseDNSForIPv6() throws UnknownHostException { String address = "ipv6.google.com"; String localhost = null; InetAddress addr = null; TableInputFormat inputFormat = new TableInputFormat(); try { localhost = InetAddress.getByName(address).getCanonicalHostName(); addr = Inet6Address.getByName(address); } catch (UnknownHostException e) { // google.com is down, we can probably forgive this test. return; } System.out.println("Should retrun the hostname for this host " + localhost + " addr : " + addr); String actualHostName = inputFormat.reverseDNS(addr); assertEquals("Should retrun the hostname for this host. Expected : " + localhost + " Actual : " + actualHostName, localhost, actualHostName); } }
@Test public void testTableInputFormatBaseReverseDNSForIPv6() throws UnknownHostException { String address = "ipv6.google.com"; String localhost = null; InetAddress addr = null; TableInputFormat inputFormat = new TableInputFormat(); try { localhost = InetAddress.getByName(address).getCanonicalHostName(); addr = Inet6Address.getByName(address); } catch (UnknownHostException e) { // google.com is down, we can probably forgive this test. return; } System.out.println("Should retrun the hostname for this host " + localhost + " addr : " + addr); String actualHostName = inputFormat.reverseDNS(addr); assertEquals("Should retrun the hostname for this host. Expected : " + localhost + " Actual : " + actualHostName, localhost, actualHostName); }
/** * Tests Number of inputSplits for MR job when specify number of mappers for TableInputFormatXXX * This test does not run MR job */ protected void testNumOfSplits(int splitsPerRegion, int expectedNumOfSplits) throws IOException, InterruptedException, ClassNotFoundException { String jobName = "TestJobForNumOfSplits"; LOG.info("Before map/reduce startup - job " + jobName); Configuration c = new Configuration(TEST_UTIL.getConfiguration()); Scan scan = new Scan(); scan.addFamily(INPUT_FAMILYS[0]); scan.addFamily(INPUT_FAMILYS[1]); c.setInt("hbase.mapreduce.tableinput.mappers.per.region", splitsPerRegion); c.set(KEY_STARTROW, ""); c.set(KEY_LASTROW, ""); Job job = Job.getInstance(c, jobName); TableMapReduceUtil.initTableMapperJob(TABLE_NAME.getNameAsString(), scan, ScanMapper.class, ImmutableBytesWritable.class, ImmutableBytesWritable.class, job); TableInputFormat tif = new TableInputFormat(); tif.setConf(job.getConfiguration()); Assert.assertEquals(TABLE_NAME, table.getName()); List<InputSplit> splits = tif.getSplits(job); Assert.assertEquals(expectedNumOfSplits, splits.size()); }
/** * Tests Number of inputSplits for MR job when specify number of mappers for TableInputFormatXXX * This test does not run MR job * * @throws IOException * @throws ClassNotFoundException * @throws InterruptedException */ public void testNumOfSplits(int splitsPerRegion, int expectedNumOfSplits) throws IOException, InterruptedException, ClassNotFoundException { String jobName = "TestJobForNumOfSplits"; LOG.info("Before map/reduce startup - job " + jobName); Configuration c = new Configuration(TEST_UTIL.getConfiguration()); Scan scan = new Scan(); scan.addFamily(INPUT_FAMILYS[0]); scan.addFamily(INPUT_FAMILYS[1]); c.setInt("hbase.mapreduce.tableinput.mappers.per.region", splitsPerRegion); c.set(KEY_STARTROW, ""); c.set(KEY_LASTROW, ""); Job job = new Job(c, jobName); TableMapReduceUtil.initTableMapperJob(TABLE_NAME.getNameAsString(), scan, ScanMapper.class, ImmutableBytesWritable.class, ImmutableBytesWritable.class, job); TableInputFormat tif = new TableInputFormat(); tif.setConf(job.getConfiguration()); Assert.assertEquals(TABLE_NAME, table.getName()); List<InputSplit> splits = tif.getSplits(job); Assert.assertEquals(expectedNumOfSplits, splits.size()); }
/** * Run MR job to test autobalance for setting number of mappers for TIF This does not run real MR * job */ protected void testAutobalanceNumOfSplit() throws IOException { // set up splits for testing List<InputSplit> splits = new ArrayList<>(5); int[] regionLen = { 10, 20, 20, 40, 60 }; for (int i = 0; i < 5; i++) { InputSplit split = new TableSplit(TABLE_NAME, new Scan(), Bytes.toBytes(i), Bytes.toBytes(i + 1), "", "", regionLen[i] * 1048576); splits.add(split); } TableInputFormat tif = new TableInputFormat(); List<InputSplit> res = tif.calculateAutoBalancedSplits(splits, 1073741824); assertEquals("Saw the wrong number of splits", 5, res.size()); TableSplit ts1 = (TableSplit) res.get(0); assertEquals("The first split end key should be", 2, Bytes.toInt(ts1.getEndRow())); TableSplit ts2 = (TableSplit) res.get(1); assertEquals("The second split regionsize should be", 20 * 1048576, ts2.getLength()); TableSplit ts3 = (TableSplit) res.get(2); assertEquals("The third split start key should be", 3, Bytes.toInt(ts3.getStartRow())); TableSplit ts4 = (TableSplit) res.get(4); assertNotEquals("The seventh split start key should not be", 4, Bytes.toInt(ts4.getStartRow())); } }
/** * Run MR job to test autobalance for setting number of mappers for TIF * This does not run real MR job */ public void testAutobalanceNumOfSplit() throws IOException { // set up splits for testing List<InputSplit> splits = new ArrayList<>(5); int[] regionLen = {10, 20, 20, 40, 60}; for (int i = 0; i < 5; i++) { InputSplit split = new TableSplit(TABLE_NAME, new Scan(), Bytes.toBytes(i), Bytes.toBytes(i + 1), "", "", regionLen[i] * 1048576); splits.add(split); } TableInputFormat tif = new TableInputFormat(); List<InputSplit> res = tif.calculateAutoBalancedSplits(splits, 1073741824); assertEquals("Saw the wrong number of splits", 5, res.size()); TableSplit ts1 = (TableSplit) res.get(0); assertEquals("The first split end key should be", 2, Bytes.toInt(ts1.getEndRow())); TableSplit ts2 = (TableSplit) res.get(1); assertEquals("The second split regionsize should be", 20 * 1048576, ts2.getLength()); TableSplit ts3 = (TableSplit) res.get(2); assertEquals("The third split start key should be", 3, Bytes.toInt(ts3.getStartRow())); TableSplit ts4 = (TableSplit) res.get(4); assertNotEquals("The seventh split start key should not be", 4, Bytes.toInt(ts4.getStartRow())); } }
tableInputFormat = new SampleIndexTableInputFormat(); } else { tableInputFormat = new TableInputFormat();