/** * Localizes resources to the given local directory. For resources on the local directory already, a symlink will be * created. Otherwise, it will be copied. * * @param resources the set of resources that need to be localized. * @param targetDir the target directory for the resources to copy / link to. */ private void copyUserResources(Map<String, LocalizeResource> resources, File targetDir) throws IOException { for (Map.Entry<String, LocalizeResource> entry : resources.entrySet()) { LocalizationUtils.localizeResource(entry.getKey(), entry.getValue(), targetDir); } }
void updateSparkEnv(Map<String, String> env) { env.put(SPARK_LIBRARY, LocalizationUtils.getLocalizedName(localizeResource.getURI())); } }
private static void untargz(File tarGzFile, File targetDir) throws IOException { try (TarArchiveInputStream tis = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(tarGzFile)))) { extractTar(tis, targetDir); } }
/** * Localizes the specified {@link LocalizeResource} in the specified {@link File targetDir} with the specified * file name and returns the {@link File} pointing to the localized file. * * @param fileName the name to localize the file with * @param resource the {@link LocalizeResource} to localize * @param targetDir the directory to localize the resource in * @return the {@link File} pointing to the localized file. */ public static File localizeResource(String fileName, LocalizeResource resource, File targetDir) throws IOException { File localizedResource = new File(targetDir, fileName); File input = getFileToLocalize(resource, targetDir); if (resource.isArchive()) { LOG.debug("Decompress file {} to {}", input, localizedResource); unpack(input, localizedResource); } else { try { LOG.debug("Hard link file from {} to {}", input, localizedResource); java.nio.file.Files.createLink(Paths.get(localizedResource.toURI()), Paths.get(input.toURI())); } catch (Exception e) { LOG.debug("Copy file from {} to {}", input, localizedResource); Files.copy(input, localizedResource); } } return localizedResource; }
private static void unpack(File archive, File targetDir) throws IOException { if (!targetDir.exists()) { //noinspection ResultOfMethodCallIgnored targetDir.mkdir(); } String extension = Files.getFileExtension(archive.getPath()).toLowerCase(); switch (extension) { case "zip": case "jar": BundleJarUtil.unJar(Locations.toLocation(archive), targetDir); break; case "gz": // gz is not recommended for archiving multiple files together. So we only support .tar.gz Preconditions.checkArgument(archive.getName().endsWith(".tar.gz"), "'.gz' format is not supported for " + "archiving multiple files. Please use 'zip', 'jar', '.tar.gz', 'tgz' or 'tar'."); untargz(archive, targetDir); break; case "tgz": untargz(archive, targetDir); break; case "tar": untar(archive, targetDir); break; default: throw new IllegalArgumentException(String.format("Unsupported compression type '%s'. Only 'zip', 'jar', " + "'tar.gz', 'tgz' and 'tar' are supported.", extension)); } }
/** * Localizes the specified {@link LocalizeResource} in the specified {@link File targetDir} with the specified * file name and returns the {@link File} pointing to the localized file. * * @param fileName the name to localize the file with * @param resource the {@link LocalizeResource} to localize * @param targetDir the directory to localize the resource in * @return the {@link File} pointing to the localized file. */ public static File localizeResource(String fileName, LocalizeResource resource, File targetDir) throws IOException { File localizedResource = new File(targetDir, fileName); File input = getFileToLocalize(resource, targetDir); if (resource.isArchive()) { LOG.debug("Decompress file {} to {}", input, localizedResource); unpack(input, localizedResource); } else { try { LOG.debug("Hard link file from {} to {}", input, localizedResource); java.nio.file.Files.createLink(Paths.get(localizedResource.toURI()), Paths.get(input.toURI())); } catch (Exception e) { LOG.debug("Copy file from {} to {}", input, localizedResource); Files.copy(input, localizedResource); } } return localizedResource; }
private static void unpack(File archive, File targetDir) throws IOException { if (!targetDir.exists()) { //noinspection ResultOfMethodCallIgnored targetDir.mkdir(); } String extension = Files.getFileExtension(archive.getPath()).toLowerCase(); switch (extension) { case "zip": case "jar": BundleJarUtil.unJar(Locations.toLocation(archive), targetDir); break; case "gz": // gz is not recommended for archiving multiple files together. So we only support .tar.gz Preconditions.checkArgument(archive.getName().endsWith(".tar.gz"), "'.gz' format is not supported for " + "archiving multiple files. Please use 'zip', 'jar', '.tar.gz', 'tgz' or 'tar'."); untargz(archive, targetDir); break; case "tgz": untargz(archive, targetDir); break; case "tar": untar(archive, targetDir); break; default: throw new IllegalArgumentException(String.format("Unsupported compression type '%s'. Only 'zip', 'jar', " + "'tar.gz', 'tgz' and 'tar' are supported.", extension)); } }
/** * Localizes resources to the given local directory. For resources on the local directory already, a symlink will be * created. Otherwise, it will be copied. * * @param resources the set of resources that need to be localized. * @param targetDir the target directory for the resources to copy / link to. */ private void copyUserResources(Map<String, LocalizeResource> resources, File targetDir) throws IOException { for (Map.Entry<String, LocalizeResource> entry : resources.entrySet()) { LocalizationUtils.localizeResource(entry.getKey(), entry.getValue(), targetDir); } }
void updateSparkEnv(Map<String, String> env) { env.put(SPARK_LIBRARY, LocalizationUtils.getLocalizedName(localizeResource.getURI())); } }
private static void untar(File tarFile, File targetDir) throws IOException { try (TarArchiveInputStream tis = new TarArchiveInputStream(new FileInputStream(tarFile))) { extractTar(tis, targetDir); } }
/** * Localizes resources to the given local directory. For resources on the local directory already, a symlink will be * created. Otherwise, it will be copied. * * @param resources the set of resources that need to be localized. * @param targetDir the target directory for the resources to copy / link to. */ private void copyUserResources(Map<String, LocalizeResource> resources, File targetDir) throws IOException { for (Map.Entry<String, LocalizeResource> entry : resources.entrySet()) { LocalizationUtils.localizeResource(entry.getKey(), entry.getValue(), targetDir); } }
void addLocalizeResource(Map<String, LocalizeResource> resources) { resources.put(LocalizationUtils.getLocalizedName(localizeResource.getURI()), localizeResource); }
private static void untargz(File tarGzFile, File targetDir) throws IOException { try (TarArchiveInputStream tis = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(tarGzFile)))) { extractTar(tis, targetDir); } }
LocalizationUtils.localizeResource(entry.getKey(), entry.getValue(), targetDir).getAbsolutePath(); } else { URI uri = entry.getValue().getURI();
void addLocalizeResource(Map<String, LocalizeResource> resources) { resources.put(LocalizationUtils.getLocalizedName(localizeResource.getURI()), localizeResource); }
private static void untar(File tarFile, File targetDir) throws IOException { try (TarArchiveInputStream tis = new TarArchiveInputStream(new FileInputStream(tarFile))) { extractTar(tis, targetDir); } }
LocalizationUtils.localizeResource(entry.getKey(), entry.getValue(), targetDir).getAbsolutePath(); } else { URI uri = entry.getValue().getURI();
void updateSparkEnv(Map<String, String> env) { env.put(SPARK_LIBRARY, LocalizationUtils.getLocalizedName(localizeResource.getURI())); } }
@Test public void testZip() throws IOException { String zipFileName = "target"; File directory = TEMP_FOLDER.newFolder("zip"); File file1 = File.createTempFile("file1", ".txt", directory); File file2 = File.createTempFile("file2", ".txt", directory); File zipFile = createZipFile(zipFileName, directory, false); File localizationDir = TEMP_FOLDER.newFolder("localZip"); File localizedResource = LocalizationUtils.localizeResource(zipFileName, new LocalizeResource(zipFile, true), localizationDir); Assert.assertTrue(localizedResource.isDirectory()); File[] files = localizedResource.listFiles(); Assert.assertNotNull(files); Assert.assertEquals(2, files.length); if (file1.getName().equals(files[0].getName())) { Assert.assertEquals(file2.getName(), files[1].getName()); } else { Assert.assertEquals(file1.getName(), files[1].getName()); Assert.assertEquals(file2.getName(), files[0].getName()); } }
void addLocalizeResource(Map<String, LocalizeResource> resources) { resources.put(LocalizationUtils.getLocalizedName(localizeResource.getURI()), localizeResource); }