@Override public OutputStream create(String url, Boolean forceCreateParentDirs) throws IOException { // TODO implement properly. return this.create(url); }
@Override public OutputStream create(String url) throws IOException { return this.create(url, false); }
@Override public boolean delete(String url, boolean isRecursiveDelete) throws IOException { try { final File file = toFile(url); if (!isRecursiveDelete && file.isDirectory()) return false; return this.delete(file); } catch (URISyntaxException e) { throw new IOException("Cannot access file.", e); } }
/** * Adjusts this instance to the properties specified in the given file. * * @param configurationUrl URL to the configuration file */ public void load(String configurationUrl) { final Optional<FileSystem> fileSystem = FileSystems.getFileSystem(configurationUrl); if (!fileSystem.isPresent()) { throw new RheemException(String.format("Could not access %s.", configurationUrl)); } try (InputStream configInputStream = fileSystem.get().open(configurationUrl)) { this.load(configInputStream); } catch (Exception e) { throw new RheemException(String.format("Could not load configuration from %s.", configurationUrl), e); } }
@Override public void updateProgress(HashMap<String, Integer> partialProgress) throws IOException { HashMap<String, Object> progressBar = new HashMap<>(); Integer overall = 0; for (String operatorName : partialProgress.keySet()) { this.progress.put(operatorName, partialProgress.get(operatorName)); } for (String operatorName: this.progress.keySet()) { overall = overall + this.progress.get(operatorName); } if (this.progress.size()>0) overall = overall/this.progress.size(); final FileSystem progressFile = FileSystems.getFileSystem(progressUrl).get(); try (final OutputStreamWriter writer = new OutputStreamWriter(progressFile.create(progressUrl, true))) { progressBar.put("overall", overall); progressBar.put("details", progress); JSONObject jsonProgress = new JSONObject(progressBar); writer.write(jsonProgress.toString()); } catch (UncheckedIOException e) { throw e.getCause(); } } }
/** * Determine the number of bytes of a given file. This method is not only a short-cut to * {@link FileSystem#getFileSize(String)} but also caches file sizes for performance reasons. * * @param fileUrl the URL of the file * @return the number of bytes of the file if it could be determined */ public static OptionalLong getFileSize(String fileUrl) { if (fileSizeCache.containsKey(fileUrl)) { return OptionalLong.of(fileSizeCache.get(fileUrl)); } final Optional<FileSystem> fileSystem = FileSystems.getFileSystem(fileUrl); if (fileSystem.isPresent()) { try { final long fileSize = fileSystem.get().getFileSize(fileUrl); fileSizeCache.put(fileUrl, fileSize); return OptionalLong.of(fileSize); } catch (FileNotFoundException e) { LOGGER.warn("Could not determine file size.", e); } } return OptionalLong.empty(); }
@Override public void doDispose() throws RheemException { Actions.doSafe(() -> { logger.info("Deleting file channel instances {}.", this.paths); final String path = this.getSinglePath(); final Optional<FileSystem> fileSystemOptional = FileSystems.getFileSystem(path); fileSystemOptional.ifPresent(fs -> { try { fs.delete(path, true); } catch (IOException e) { throw new UncheckedIOException(e); } }); }); } }
public static FileSystem requireFileSystem(String fileUrl) { return getFileSystem(fileUrl).orElseThrow( () -> new RheemException(String.format("Could not identify filesystem for \"%s\".", fileUrl)) ); }
@Override public void setProperties(Configuration configuration) { configuration.load(ReflectionUtils.loadResource(RHEEM_BASIC_DEFAULTS_PROPERTIES)); final File localTempDir = LocalFileSystem.findTempDir(); if (localTempDir != null) { configuration.setProperty(TEMP_DIR_PROPERTY, LocalFileSystem.toURL(localTempDir)); } }
@Override public void setProperties(Configuration configuration) { final File localTempDir = LocalFileSystem.findTempDir(); if (localTempDir != null) { configuration.setProperty("rheem.graphchi.tempdir", localTempDir.toString()); } }
@Override public boolean isDirectory(String url) { try { return toFile(url).isDirectory(); } catch (URISyntaxException | MalformedURLException e) { logger.warn("Could not inspect directory.", e); return false; } }
@Override public boolean delete(String url, boolean isRecursiveDelete) throws IOException { return this.getHdfs(url).delete(new Path(url), isRecursiveDelete); } }
public static Optional<FileSystem> getFileSystem(String fileUrl) { return registeredFileSystems.stream() .filter(fileSystem -> fileSystem.canHandle(fileUrl)) .findAny(); }
/** * As {@link #findActualInputPaths(String)} but requires the presence of only a single input file. */ public static String findActualSingleInputPath(String ostensibleInputFile) { final Collection<String> inputPaths = FileSystems.findActualInputPaths(ostensibleInputFile); if (inputPaths.size() != 1) { throw new RheemException(String.format( "Illegal number of files for \"%s\": %s", ostensibleInputFile, inputPaths )); // TODO: Add support. } return inputPaths.iterator().next(); }
/** * Create an empty file. * * @param file that should be created */ public static void touch(File file) { ensureDir(file.getParentFile()); try (FileOutputStream fos = new FileOutputStream(file)) { } catch (IOException e) { throw new RheemException(String.format("Could not create %s.", file), e); } }
@Override public InputStream open(String url) throws IOException { try { File file = toFile(url); return new FileInputStream(file); } catch (URISyntaxException e) { throw new IOException("Could not process the given URL.", e); } }
@Override public OutputStream create(String url) throws IOException { return this.getHdfs(url).create(new Path(url), true); }
@Override public long getFileSize(String fileUrl) throws FileNotFoundException { try { File file = toFile(fileUrl); return file.length(); } catch (MalformedURLException | URISyntaxException e) { logger.error("Illegal URL: \"{}\"", fileUrl); throw new FileNotFoundException("File not found, because the URL is not correct."); } }
@Override public InputStream open(String url) throws IOException { return this.getHdfs(url).open(new Path(url)); }
private static File toFile(String fileUrl) throws URISyntaxException, MalformedURLException { if (fileUrl.startsWith("file:")) { return new File(new URL(fileUrl).toURI()); } else { logger.warn("Expect URLs, but got {}. Converting it to file:{}...", fileUrl, fileUrl); return toFile("file:" + fileUrl); } }