@Override public void onRemoval(RemovalNotification<String, ISource> entry) { ISource s = entry.getValue(); if (s != null) { try { s.close(); } catch (Throwable e) { logger.error("Failed to close ISource: {}", s.getClass().getName(), e); } } } }).build();
@Test public void testGenSqoopCmd_Partition() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_inner_join_cube"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue(cmd.contains( "--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"")); source.close(); }
@Test public void testGenSqoopCmd_WithLookupShardBy() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_jdbc_shard"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue(cmd.contains( "--boundary-query \"SELECT MIN(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\"), MAX(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_CATEGORY_GROUPINGS\\\" AS \\\"TEST_CATEGORY_GROUPINGS\\\"\"")); source.close(); }
@Test public void testGenSqoopCmd_NoPartition() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_left_join_cube"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(0L, Long.MAX_VALUE)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue( cmd.contains("--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"\"")); source.close(); }
@Test public void testBasics() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); ISourceMetadataExplorer explorer = source.getSourceMetadataExplorer(); ISampleDataDeployer deployer = source.getSampleDataDeployer(); Assert.assertTrue(source instanceof JdbcSource); Assert.assertTrue(explorer instanceof JdbcExplorer); Assert.assertTrue(deployer instanceof JdbcExplorer); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); Class adaptTo = Object.class; expectedCannotAdaptEx.expect(RuntimeException.class); expectedCannotAdaptEx.expectMessage("Cannot adapt to " + adaptTo); source.adaptToBuildEngine(adaptTo); TableMetadataManager tblManager = TableMetadataManager.getInstance(getTestConfig()); IReadableTable table = source.createReadableTable(tblManager.getTableDesc("test_kylin_fact", "default"), null); Assert.assertTrue(table instanceof JdbcTable); source.close(); } }
@Override public void onRemoval(RemovalNotification<String, ISource> entry) { ISource s = entry.getValue(); if (s != null) { try { s.close(); } catch (Throwable e) { logger.error("Failed to close ISource: {}", s.getClass().getName(), e); } } } }).build();