@Override public LoadSpecResult loadSegment(final File outDir) throws SegmentLoadingException { return new LoadSpecResult(puller.getSegmentFiles(path.toFile(), outDir).size()); } }
@Override public InputStream getInputStream(URI uri) throws IOException { return buildFileObject(uri).openInputStream(); }
@VisibleForTesting public void getSegmentFiles(DataSegment segment, File dir) throws SegmentLoadingException { getSegmentFiles(getFile(segment), dir); }
@Before public void setup() throws IOException { tmpDir = temporaryFolder.newFolder(); puller = new LocalDataSegmentPuller(); }
Files.asByteSource(oldFile), new File(dir, oldFile.getName()), shouldRetryPredicate(), DEFAULT_RETRY_COUNT ).getFiles() Files.asByteSource(sourceFile), dir, shouldRetryPredicate(), false ); Files.asByteSource(sourceFile), outFile, shouldRetryPredicate() ); log.info(
public SegmentLoaderLocalCacheManagerTest(SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) { jsonMapper = new DefaultObjectMapper(); jsonMapper.registerSubtypes(new NamedType(LocalLoadSpec.class, "local")); jsonMapper.setInjectableValues( new InjectableValues.Std().addValue( LocalDataSegmentPuller.class, new LocalDataSegmentPuller() ) ); this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; }
Files.asByteSource(oldFile), new File(dir, oldFile.getName()), shouldRetryPredicate(), DEFAULT_RETRY_COUNT ).getFiles() Files.asByteSource(sourceFile), dir, shouldRetryPredicate(), false ); Files.asByteSource(sourceFile), outFile, shouldRetryPredicate() ); log.info(
@Test public void simpleDirectoryTest() throws IOException, SegmentLoadingException { File srcDir = temporaryFolder.newFolder(); File tmpFile = File.createTempFile("test", "file", srcDir); File expectedOutput = new File(tmpDir, Files.getNameWithoutExtension(tmpFile.getAbsolutePath())); Assert.assertFalse(expectedOutput.exists()); puller.getSegmentFiles(srcDir, tmpDir); Assert.assertTrue(expectedOutput.exists()); } }
@VisibleForTesting public void getSegmentFiles(DataSegment segment, File dir) throws SegmentLoadingException { getSegmentFiles(getFile(segment), dir); }
.addValue(IndexingServiceClient.class, new NoopIndexingServiceClient()) .addValue(AuthorizerMapper.class, new AuthorizerMapper(ImmutableMap.of())) .addValue(LocalDataSegmentPuller.class, new LocalDataSegmentPuller()) );
/** * Returns the "version" (aka last modified timestamp) of the URI of interest * * @param uri The URI to check the last modified timestamp * * @return The last modified timestamp in ms of the URI in String format */ @Override public String getVersion(URI uri) { return StringUtils.format("%d", buildFileObject(uri).getLastModified()); }
@Test public void simpleGZTest() throws IOException, SegmentLoadingException { File zipFile = File.createTempFile("gztest", ".gz"); File unZipFile = new File( tmpDir, Files.getNameWithoutExtension( zipFile.getAbsolutePath() ) ); unZipFile.delete(); zipFile.delete(); try (OutputStream fOutStream = new FileOutputStream(zipFile)) { try (OutputStream outputStream = new GZIPOutputStream(fOutStream)) { outputStream.write(new byte[0]); outputStream.flush(); } } Assert.assertTrue(zipFile.exists()); Assert.assertFalse(unZipFile.exists()); puller.getSegmentFiles(zipFile, tmpDir); Assert.assertTrue(unZipFile.exists()); }
@Override public InputStream getInputStream(URI uri) throws IOException { return buildFileObject(uri).openInputStream(); }
@Test public void simpleZipTest() throws IOException, SegmentLoadingException { File file = new File(tmpDir, "test1data"); File zipFile = temporaryFolder.newFile("ziptest.zip"); try (OutputStream outputStream = new FileOutputStream(file)) { outputStream.write(new byte[0]); outputStream.flush(); } CompressionUtils.zip(tmpDir, zipFile); file.delete(); Assert.assertFalse(file.exists()); Assert.assertTrue(zipFile.exists()); puller.getSegmentFiles(zipFile, tmpDir); Assert.assertTrue(file.exists()); }
/** * Returns the "version" (aka last modified timestamp) of the URI of interest * * @param uri The URI to check the last modified timestamp * * @return The last modified timestamp in ms of the URI in String format */ @Override public String getVersion(URI uri) { return StringUtils.format("%d", buildFileObject(uri).getLastModified()); }
@Override public LoadSpecResult loadSegment(final File outDir) throws SegmentLoadingException { return new LoadSpecResult(puller.getSegmentFiles(path.toFile(), outDir).size()); } }