@Override public Segment getSegment(DataSegment segment) throws SegmentLoadingException { File segmentFiles = getSegmentFiles(segment); File factoryJson = new File(segmentFiles, "factory.json"); final SegmentizerFactory factory; if (factoryJson.exists()) { try { factory = jsonMapper.readValue(factoryJson, SegmentizerFactory.class); } catch (IOException e) { throw new SegmentLoadingException(e, "%s", e.getMessage()); } } else { factory = new MMappedQueryableSegmentizerFactory(indexIO); } return factory.factorize(segment, segmentFiles); }
@Test public void testGetAndCleanSegmentFiles() throws Exception { final File localStorageFolder = tmpFolder.newFolder("local_storage_folder"); final DataSegment segmentToDownload = dataSegmentWithInterval("2014-10-20T00:00:00Z/P1D").withLoadSpec( ImmutableMap.of( "type", "local", "path", localStorageFolder.getCanonicalPath() + "/test_segment_loader" + "/2014-10-20T00:00:00.000Z_2014-10-21T00:00:00.000Z/2015-05-27T03:38:35.683Z" + "/0/index.zip" ) ); // manually create a local segment under localStorageFolder final File localSegmentFile = new File( localStorageFolder, "test_segment_loader/2014-10-20T00:00:00.000Z_2014-10-21T00:00:00.000Z/2015-05-27T03:38:35.683Z/0" ); localSegmentFile.mkdirs(); final File indexZip = new File(localSegmentFile, "index.zip"); indexZip.createNewFile(); Assert.assertFalse("Expect cache miss before downloading segment", manager.isSegmentLoaded(segmentToDownload)); manager.getSegmentFiles(segmentToDownload); Assert.assertTrue("Expect cache hit after downloading segment", manager.isSegmentLoaded(segmentToDownload)); manager.cleanup(segmentToDownload); Assert.assertFalse("Expect cache miss after dropping segment", manager.isSegmentLoaded(segmentToDownload)); }
@Test public void testFetchSegments() throws SegmentLoadingException, IOException { File expectedFile = temporaryFolder.newFile(); EasyMock .expect(mockSegmentLoaderLocalCacheManager.getSegmentFiles(EasyMock.anyObject())) .andReturn(expectedFile).anyTimes(); EasyMock .expect(mockSegmentLoaderLocalCacheManager.withConfig(EasyMock.anyObject())) .andReturn(mockSegmentLoaderLocalCacheManager).anyTimes(); EasyMock.replay(mockSegmentLoaderLocalCacheManager); DataSegment dataSegment = DataSegment.builder().dataSource("source").interval(Intervals.of("2012-01-01/P1D")).version("1").size(1).build(); List<DataSegment> segments = ImmutableList.of ( dataSegment ); Map actualFetchedSegment = taskToolbox.build(task).fetchSegments(segments); Assert.assertEquals(expectedFile, actualFetchedSegment.get(dataSegment)); } @Test
File segmentFile = manager.getSegmentFiles(segmentToDownload); Assert.assertTrue(segmentFile.getAbsolutePath().contains("/local_storage_folder/")); Assert.assertTrue("Expect cache hit after downloading segment", manager.isSegmentLoaded(segmentToDownload)); indexZip2.createNewFile(); File segmentFile2 = manager.getSegmentFiles(segmentToDownload2); Assert.assertTrue(segmentFile2.getAbsolutePath().contains("/local_storage_folder2/")); Assert.assertTrue("Expect cache hit after downloading segment", manager.isSegmentLoaded(segmentToDownload2));
File segmentFile = manager.getSegmentFiles(segmentToDownload); Assert.assertTrue(segmentFile.getAbsolutePath().contains("/local_storage_folder2/")); Assert.assertTrue("Expect cache hit after downloading segment", manager.isSegmentLoaded(segmentToDownload));
manager.getSegmentFiles(segmentToDownload); Assert.fail();
File segmentFile = manager.getSegmentFiles(segmentToDownload); Assert.assertTrue(segmentFile.getAbsolutePath().contains("/local_storage_folder/")); Assert.assertTrue("Expect cache hit after downloading segment", manager.isSegmentLoaded(segmentToDownload));
@Override public Segment getSegment(DataSegment segment) throws SegmentLoadingException { File segmentFiles = getSegmentFiles(segment); File factoryJson = new File(segmentFiles, "factory.json"); final SegmentizerFactory factory; if (factoryJson.exists()) { try { factory = jsonMapper.readValue(factoryJson, SegmentizerFactory.class); } catch (IOException e) { throw new SegmentLoadingException(e, "%s", e.getMessage()); } } else { factory = new MMappedQueryableSegmentizerFactory(indexIO); } return factory.factorize(segment, segmentFiles); }