@Test public void testPathFilter() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( TEST_FILES, RETURNED_PATH_DOMAIN); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, RETURNED_PATH_DOMAIN); backgroundHiveSplitLoader.start(hiveSplitSource); List<String> paths = drain(hiveSplitSource); assertEquals(paths.size(), 1); assertEquals(paths.get(0), RETURNED_PATH.toString()); }
@Test public void testPathFilterOneBucketMatchPartitionedTable() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( TEST_FILES, RETURNED_PATH_DOMAIN, Optional.of(new HiveBucketFilter(ImmutableSet.of(0, 1))), PARTITIONED_TABLE, Optional.of(new HiveBucketHandle(BUCKET_COLUMN_HANDLES, BUCKET_COUNT, BUCKET_COUNT))); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, RETURNED_PATH_DOMAIN); backgroundHiveSplitLoader.start(hiveSplitSource); List<String> paths = drain(hiveSplitSource); assertEquals(paths.size(), 1); assertEquals(paths.get(0), RETURNED_PATH.toString()); }
@Test public void testEmptyFileWithNoBlocks() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( ImmutableList.of(locatedFileStatusWithNoBlocks(RETURNED_PATH)), TupleDomain.none()); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, TupleDomain.none()); backgroundHiveSplitLoader.start(hiveSplitSource); List<HiveSplit> splits = drainSplits(hiveSplitSource); assertEquals(splits.size(), 1); assertEquals(splits.get(0).getPath(), RETURNED_PATH.toString()); assertEquals(splits.get(0).getLength(), 0); }
@Test public void testPathFilterBucketedPartitionedTable() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( TEST_FILES, RETURNED_PATH_DOMAIN, Optional.empty(), PARTITIONED_TABLE, Optional.of( new HiveBucketHandle( getRegularColumnHandles(PARTITIONED_TABLE), BUCKET_COUNT, BUCKET_COUNT))); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, RETURNED_PATH_DOMAIN); backgroundHiveSplitLoader.start(hiveSplitSource); List<String> paths = drain(hiveSplitSource); assertEquals(paths.size(), 1); assertEquals(paths.get(0), RETURNED_PATH.toString()); }
@Test public void testNoPathFilter() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( TEST_FILES, TupleDomain.none()); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, TupleDomain.none()); backgroundHiveSplitLoader.start(hiveSplitSource); assertEquals(drain(hiveSplitSource).size(), 2); }
@Test public void testNoHangIfPartitionIsOffline() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoaderOfflinePartitions(); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, TupleDomain.all()); backgroundHiveSplitLoader.start(hiveSplitSource); assertThrows(RuntimeException.class, () -> drain(hiveSplitSource)); assertThrows(RuntimeException.class, () -> hiveSplitSource.isFinished()); }
@Test public void testPathFilter() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( TEST_FILES, RETURNED_PATH_DOMAIN); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, RETURNED_PATH_DOMAIN); backgroundHiveSplitLoader.start(hiveSplitSource); List<String> paths = drain(hiveSplitSource); assertEquals(paths.size(), 1); assertEquals(paths.get(0), RETURNED_PATH.toString()); }
@Test public void testPathFilterOneBucketMatchPartitionedTable() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( TEST_FILES, RETURNED_PATH_DOMAIN, Optional.of(new HiveBucketFilter(ImmutableSet.of(0, 1))), PARTITIONED_TABLE, Optional.of(new HiveBucketHandle(BUCKET_COLUMN_HANDLES, BUCKET_COUNT, BUCKET_COUNT))); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, RETURNED_PATH_DOMAIN); backgroundHiveSplitLoader.start(hiveSplitSource); List<String> paths = drain(hiveSplitSource); assertEquals(paths.size(), 1); assertEquals(paths.get(0), RETURNED_PATH.toString()); }
@Test public void testNoPathFilter() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( TEST_FILES, TupleDomain.none()); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, TupleDomain.none()); backgroundHiveSplitLoader.start(hiveSplitSource); assertEquals(drain(hiveSplitSource).size(), 2); }
@Test public void testEmptyFileWithNoBlocks() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( ImmutableList.of(locatedFileStatusWithNoBlocks(RETURNED_PATH)), TupleDomain.none()); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, TupleDomain.none()); backgroundHiveSplitLoader.start(hiveSplitSource); List<HiveSplit> splits = drainSplits(hiveSplitSource); assertEquals(splits.size(), 1); assertEquals(splits.get(0).getPath(), RETURNED_PATH.toString()); assertEquals(splits.get(0).getLength(), 0); }
@Test public void testPathFilterBucketedPartitionedTable() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( TEST_FILES, RETURNED_PATH_DOMAIN, Optional.empty(), PARTITIONED_TABLE, Optional.of( new HiveBucketHandle( getRegularColumnHandles(PARTITIONED_TABLE), BUCKET_COUNT, BUCKET_COUNT))); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, RETURNED_PATH_DOMAIN); backgroundHiveSplitLoader.start(hiveSplitSource); List<String> paths = drain(hiveSplitSource); assertEquals(paths.size(), 1); assertEquals(paths.get(0), RETURNED_PATH.toString()); }
@Test public void testNoHangIfPartitionIsOffline() throws Exception { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoaderOfflinePartitions(); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader, TupleDomain.all()); backgroundHiveSplitLoader.start(hiveSplitSource); assertThrows(RuntimeException.class, () -> drain(hiveSplitSource)); assertThrows(RuntimeException.class, () -> hiveSplitSource.isFinished()); }