/** * Returns a new path by appending an extension. * * @param extractFrom the path to extract the extension from * @param appendTo the path to append the extension to */ public static String addExtension(String extractFrom, String appendTo) { String suffix = getExtension(extractFrom); if (!suffix.isEmpty()) { return appendTo + '.' + suffix; } return appendTo; }
createRuntimeConfigJar(runtimeConfigDir, localFiles); } finally { Paths.deleteRecursively(runtimeConfigDir);
/** * Based on the given {@link TwillSpecification}, upload LocalFiles to Yarn Cluster. * @param twillSpec The {@link TwillSpecification} for populating resource. */ private Multimap<String, LocalFile> populateRunnableLocalFiles(TwillSpecification twillSpec) throws IOException { Multimap<String, LocalFile> localFiles = HashMultimap.create(); LOG.debug("Populating Runnable LocalFiles"); for (Map.Entry<String, RuntimeSpecification> entry: twillSpec.getRunnables().entrySet()) { String runnableName = entry.getKey(); for (LocalFile localFile : entry.getValue().getLocalFiles()) { Location location; URI uri = localFile.getURI(); if (appLocation.toURI().getScheme().equals(uri.getScheme())) { // If the source file location is having the same scheme as the target location, no need to copy location = appLocation.getLocationFactory().create(uri); } else { URL url = uri.toURL(); LOG.debug("Create and copy {} : {}", runnableName, url); // Preserves original suffix for expansion. location = copyFromURL(url, createTempLocation(Paths.addExtension(url.getFile(), localFile.getName()))); LOG.debug("Done {} : {}", runnableName, url); } localFiles.put(runnableName, new DefaultLocalFile(localFile.getName(), location.toURI(), location.lastModified(), location.length(), localFile.isArchive(), localFile.getPattern())); } } LOG.debug("Done Runnable LocalFiles"); return localFiles; }
createRuntimeConfigJar(runtimeConfigDir, localFiles); } finally { Paths.deleteRecursively(runtimeConfigDir);
/** * Based on the given {@link TwillSpecification}, upload LocalFiles to Yarn Cluster. * @param twillSpec The {@link TwillSpecification} for populating resource. */ private Multimap<String, LocalFile> populateRunnableLocalFiles(TwillSpecification twillSpec) throws IOException { Multimap<String, LocalFile> localFiles = HashMultimap.create(); LOG.debug("Populating Runnable LocalFiles"); for (Map.Entry<String, RuntimeSpecification> entry: twillSpec.getRunnables().entrySet()) { String runnableName = entry.getKey(); for (LocalFile localFile : entry.getValue().getLocalFiles()) { Location location; URI uri = localFile.getURI(); if (appLocation.toURI().getScheme().equals(uri.getScheme())) { // If the source file location is having the same scheme as the target location, no need to copy location = appLocation.getLocationFactory().create(uri); } else { URL url = uri.toURL(); LOG.debug("Create and copy {} : {}", runnableName, url); // Preserves original suffix for expansion. location = copyFromURL(url, createTempLocation(Paths.addExtension(url.getFile(), localFile.getName()))); LOG.debug("Done {} : {}", runnableName, url); } localFiles.put(runnableName, new DefaultLocalFile(localFile.getName(), location.toURI(), location.lastModified(), location.length(), localFile.isArchive(), localFile.getPattern())); } } LOG.debug("Done Runnable LocalFiles"); return localFiles; }
/** * Returns a new path by appending an extension. * * @param extractFrom the path to extract the extension from * @param appendTo the path to append the extension to */ public static String addExtension(String extractFrom, String appendTo) { String suffix = getExtension(extractFrom); if (!suffix.isEmpty()) { return appendTo + '.' + suffix; } return appendTo; }
createRuntimeConfigJar(runtimeConfigDir, localFiles, stagingDir); } finally { Paths.deleteRecursively(runtimeConfigDir);
private Location createTempLocation(String fileName) { String name; String suffix = Paths.getExtension(fileName); name = fileName.substring(0, fileName.length() - suffix.length() - 1); try { return appLocation.append(name).getTempFile('.' + suffix); } catch (IOException e) { throw Throwables.propagate(e); } }
createRuntimeConfigJar(runtimeConfigDir, localFiles, stagingDir); } finally { Paths.deleteRecursively(runtimeConfigDir);
private Location createTempLocation(String fileName) { String name; String suffix = Paths.getExtension(fileName); name = fileName.substring(0, fileName.length() - suffix.length() - 1); try { return appLocation.append(name).getTempFile('.' + suffix); } catch (IOException e) { throw Throwables.propagate(e); } }
@Override public Location get(String name, Loader loader) throws IOException { String suffix = Paths.getExtension(name); String prefix = name.substring(0, name.length() - suffix.length() - 1); Location targetLocation = baseDir.append(prefix).getTempFile('.' + suffix); loader.load(name, targetLocation); return targetLocation; } }
@Override public Location get(String name, Loader loader) throws IOException { String suffix = Paths.getExtension(name); String prefix = name.substring(0, name.length() - suffix.length() - 1); Location targetLocation = baseDir.append(prefix).getTempFile('.' + suffix); loader.load(name, targetLocation); return targetLocation; } }
private LocalFile resolveLocalFile(LocalFile localFile, Path stagingDir) throws IOException { URI uri = localFile.getURI(); String scheme = uri.getScheme(); // If local file, resolve the last modified time and the file size if (scheme == null || "file".equals(scheme)) { File file = new File(uri.getPath()); return new DefaultLocalFile(localFile.getName(), uri, file.lastModified(), file.length(), localFile.isArchive(), localFile.getPattern()); } // If have the same scheme as the location factory, resolve time and size using Location if (Objects.equals(locationFactory.getHomeLocation().toURI().getScheme(), scheme)) { Location location = locationFactory.create(uri); return new DefaultLocalFile(localFile.getName(), uri, location.lastModified(), location.length(), localFile.isArchive(), localFile.getPattern()); } // For other cases, attempt to save the URI content to local file, using support URLSteamHandler try (InputStream input = uri.toURL().openStream()) { Path tempFile = Files.createTempFile(stagingDir, localFile.getName(), Paths.getExtension(localFile.getName())); Files.copy(input, tempFile, StandardCopyOption.REPLACE_EXISTING); BasicFileAttributes attrs = Files.readAttributes(tempFile, BasicFileAttributes.class); return new DefaultLocalFile(localFile.getName(), tempFile.toUri(), attrs.lastModifiedTime().toMillis(), attrs.size(), localFile.isArchive(), localFile.getPattern()); } }
private LocalFile resolveLocalFile(LocalFile localFile, Path stagingDir) throws IOException { URI uri = localFile.getURI(); String scheme = uri.getScheme(); // If local file, resolve the last modified time and the file size if (scheme == null || "file".equals(scheme)) { File file = new File(uri.getPath()); return new DefaultLocalFile(localFile.getName(), uri, file.lastModified(), file.length(), localFile.isArchive(), localFile.getPattern()); } // If have the same scheme as the location factory, resolve time and size using Location if (Objects.equals(locationFactory.getHomeLocation().toURI().getScheme(), scheme)) { Location location = locationFactory.create(uri); return new DefaultLocalFile(localFile.getName(), uri, location.lastModified(), location.length(), localFile.isArchive(), localFile.getPattern()); } // For other cases, attempt to save the URI content to local file, using support URLSteamHandler try (InputStream input = uri.toURL().openStream()) { Path tempFile = Files.createTempFile(stagingDir, localFile.getName(), Paths.getExtension(localFile.getName())); Files.copy(input, tempFile, StandardCopyOption.REPLACE_EXISTING); BasicFileAttributes attrs = Files.readAttributes(tempFile, BasicFileAttributes.class); return new DefaultLocalFile(localFile.getName(), tempFile.toUri(), attrs.lastModifiedTime().toMillis(), attrs.size(), localFile.isArchive(), localFile.getPattern()); } }