@Test(expected = SegmentLoadingException.class) public void testFindSegmentsFail() throws SegmentLoadingException { // remove one of index.zip while keeping its descriptor.json indexZip4_1.delete(); final LocalDataSegmentFinder localDataSegmentFinder = new LocalDataSegmentFinder(mapper); localDataSegmentFinder.findSegments(dataSourceDir.getAbsolutePath(), false); }
@Override public Set<DataSegment> findSegments(String workingDirPath, boolean updateDescriptor) throws SegmentLoadingException { final Map<SegmentId, Pair<DataSegment, Long>> timestampedSegments = new HashMap<>(); final File workingDir = new File(workingDirPath); if (!workingDir.isDirectory()) { throw new SegmentLoadingException("Working directory [%s] didn't exist !?", workingDir); } recursiveSearchSegments(timestampedSegments, workingDir, updateDescriptor); return timestampedSegments.values().stream().map(x -> x.lhs).collect(Collectors.toSet()); }
@Test public void testPreferNewestSegment() throws Exception { dataSourceDir = temporaryFolder.newFolder(); descriptor1 = new File(dataSourceDir.getAbsolutePath() + "/interval10/v10/0/older", DESCRIPTOR_JSON); descriptor2 = new File(dataSourceDir.getAbsolutePath() + "/interval10/v10/0/newer", DESCRIPTOR_JSON); descriptor1.getParentFile().mkdirs(); descriptor2.getParentFile().mkdirs(); mapper.writeValue(descriptor1, SEGMENT_1); mapper.writeValue(descriptor2, SEGMENT_1); indexZip1 = new File(descriptor1.getParentFile(), INDEX_ZIP); indexZip2 = new File(descriptor2.getParentFile(), INDEX_ZIP); FileOutputStream fos1 = new FileOutputStream(indexZip1); fos1.getFD().sync(); fos1.close(); Thread.sleep(1000); FileOutputStream fos2 = new FileOutputStream(indexZip2); fos2.getFD().sync(); fos2.close(); final Set<DataSegment> segments = new LocalDataSegmentFinder(mapper).findSegments( dataSourceDir.getAbsolutePath(), false ); Assert.assertEquals(1, segments.size()); Assert.assertEquals(indexZip2.getAbsolutePath(), segments.iterator().next().getLoadSpec().get("path")); }
recursiveSearchSegments(timestampedSegments, file, updateDescriptor); } else if ("descriptor.json".equals(file.getName())) { final File indexZip = new File(file.getParentFile(), "index.zip");
@Test public void testFindSegments() throws SegmentLoadingException, IOException final LocalDataSegmentFinder localDataSegmentFinder = new LocalDataSegmentFinder(mapper); final Set<DataSegment> segments = localDataSegmentFinder.findSegments(dataSourceDir.getAbsolutePath(), false); final Set<DataSegment> segments2 = localDataSegmentFinder.findSegments(dataSourceDir.getAbsolutePath(), true);
@Override public Set<DataSegment> findSegments(String workingDirPath, boolean updateDescriptor) throws SegmentLoadingException { final Map<String, Pair<DataSegment, Long>> timestampedSegments = new HashMap<>(); final File workingDir = new File(workingDirPath); if (!workingDir.isDirectory()) { throw new SegmentLoadingException("Working directory [%s] didn't exist !?", workingDir); } recursiveSearchSegments(timestampedSegments, workingDir, updateDescriptor); return timestampedSegments.values().stream().map(x -> x.lhs).collect(Collectors.toSet()); }
recursiveSearchSegments(timestampedSegments, file, updateDescriptor); } else if ("descriptor.json".equals(file.getName())) { final File indexZip = new File(file.getParentFile(), "index.zip");