/** * Adds a suffix to the final name in the path. * * @param suffix The suffix to be added * @return the new path including the suffix */ public Path suffix(String suffix) { return new Path(getParent(), getName() + suffix); }
private InflaterInputStreamFactory<?> getInflaterInputStreamFactory(Path path) { String fileExtension = extractFileExtension(path.getName()); if (fileExtension != null) { return getInflaterInputStreamFactory(fileExtension); } else { return null; } }
/** * A simple hook to filter files and directories from the input. * The method may be overridden. Hadoop's FileInputFormat has a similar mechanism and applies the * same filters by default. * * @param fileStatus The file status to check. * @return true, if the given file or directory is accepted */ public boolean acceptFile(FileStatus fileStatus) { final String name = fileStatus.getPath().getName(); return !name.startsWith("_") && !name.startsWith(".") && !filesFilter.filterPath(fileStatus.getPath()); }
private void rollPartFile(final long currentTime) throws IOException { closePartFile(); final Path partFilePath = assembleNewPartPath(); final RecoverableFsDataOutputStream stream = fsWriter.open(partFilePath); inProgressPart = partFileFactory.openNew(bucketId, stream, partFilePath, currentTime); if (LOG.isDebugEnabled()) { LOG.debug("Subtask {} opening new part file \"{}\" for bucket id={}.", subtaskIndex, partFilePath.getName(), bucketId); } partCounter++; }
/** * This recreates the new working directory of the recovered RocksDB instance and links/copies the contents from * a local state. */ private void restoreInstanceDirectoryFromPath(Path source) throws IOException { FileSystem fileSystem = source.getFileSystem(); final FileStatus[] fileStatuses = fileSystem.listStatus(source); if (fileStatuses == null) { throw new IOException("Cannot list file statues. Directory " + source + " does not exist."); } for (FileStatus fileStatus : fileStatuses) { final Path filePath = fileStatus.getPath(); final String fileName = filePath.getName(); File restoreFile = new File(source.getPath(), fileName); File targetFile = new File(stateBackend.instanceRocksDBPath.getPath(), fileName); if (fileName.endsWith(SST_FILE_SUFFIX)) { // hardlink'ing the immutable sst-files. Files.createLink(targetFile.toPath(), restoreFile.toPath()); } else { // true copy for all other files. Files.copy(restoreFile.toPath(), targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } } }
private List<String> listNewFiles(FileSystem fileSystem) throws IOException { List<String> files = new ArrayList<String>(); FileStatus[] statuses = fileSystem.listStatus(new Path(path)); if (statuses == null) { LOG.warn("Path does not exist: {}", path); } else { for (FileStatus status : statuses) { Path filePath = status.getPath(); String fileName = filePath.getName(); long modificationTime = status.getModificationTime(); if (!isFiltered(fileName, modificationTime)) { files.add(filePath.toString()); modificationTimes.put(fileName, modificationTime); } } } return files; }
private static void getCopyTasks(Path p, String rel, List<FileCopyTask> tasks) throws IOException { FileStatus[] res = p.getFileSystem().listStatus(p); if (res == null) { return; } for (FileStatus fs : res) { if (fs.isDir()) { getCopyTasks(fs.getPath(), rel + fs.getPath().getName() + "/", tasks); } else { Path cp = fs.getPath(); tasks.add(new FileCopyTask(cp, rel + cp.getName())); } } } }
for (FileStatus jobArchive : jobArchives) { Path jobArchivePath = jobArchive.getPath(); String jobID = jobArchivePath.getName(); try { JobID.fromHexString(jobID);
jarFileNames.add(jar.getName()); filesToUpload.add(new FileUpload(Paths.get(jar.toUri()), RestConstants.CONTENT_TYPE_JAR)); artifactFileNames.add(new JobSubmitRequestBody.DistributedCacheFile(artifacts.getKey(), new Path(artifacts.getValue().filePath).getName())); filesToUpload.add(new FileUpload(Paths.get(artifacts.getValue().filePath), RestConstants.CONTENT_TYPE_BINARY));
copyFile(planPath, targetDir, planPath.getName()); for (String file : filesToCopy) { Path source = new Path(file); copyFile(source, targetDir, source.getName()); String planNameWithExtension = planPath.getName(); String planName = planNameWithExtension.substring(0, planNameWithExtension.indexOf(".py"));
@Test public void testGetName() { Path p = new Path("/my/fancy/path"); assertEquals("path", p.getName()); p = new Path("/my/fancy/path/"); assertEquals("path", p.getName()); p = new Path("hdfs:///my/path"); assertEquals("path", p.getName()); p = new Path("hdfs:///myPath/"); assertEquals("myPath", p.getName()); p = new Path("/"); assertEquals("", p.getName()); p = new Path("C:/my/windows/path"); assertEquals("path", p.getName()); p = new Path("file:/C:/my/windows/path"); assertEquals("path", p.getName()); }
@Test public void testCloseWithNoData() throws Exception { final RecoverableWriter writer = getNewFileSystemWriter(); final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); final RecoverableFsDataOutputStream stream = writer.open(path); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertTrue(fileContents.getKey().getName().startsWith(".part-0.inprogress.")); Assert.assertTrue(fileContents.getValue().isEmpty()); } stream.closeForCommit().commit(); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertTrue(fileContents.getValue().isEmpty()); } }
public static void addUserArtifactEntries(Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> userArtifacts, JobGraph jobGraph) { if (!userArtifacts.isEmpty()) { try { java.nio.file.Path tmpDir = Files.createTempDirectory("flink-distributed-cache-" + jobGraph.getJobID()); for (Tuple2<String, DistributedCache.DistributedCacheEntry> originalEntry : userArtifacts) { Path filePath = new Path(originalEntry.f1.filePath); boolean isLocalDir = false; try { FileSystem sourceFs = filePath.getFileSystem(); isLocalDir = !sourceFs.isDistributedFS() && sourceFs.getFileStatus(filePath).isDir(); } catch (IOException ioe) { LOG.warn("Could not determine whether {} denotes a local path.", filePath, ioe); } // zip local directories because we only support file uploads DistributedCache.DistributedCacheEntry entry; if (isLocalDir) { Path zip = FileUtils.compressDirectory(filePath, new Path(tmpDir.toString(), filePath.getName() + ".zip")); entry = new DistributedCache.DistributedCacheEntry(zip.toString(), originalEntry.f1.isExecutable, true); } else { entry = new DistributedCache.DistributedCacheEntry(filePath.toString(), originalEntry.f1.isExecutable, false); } jobGraph.addUserArtifact(originalEntry.f0, entry); } } catch (IOException ioe) { throw new FlinkRuntimeException("Could not compress distributed-cache artifacts.", ioe); } } }
@Test public void testCommitAfterNormalClose() throws Exception { final RecoverableWriter writer = getNewFileSystemWriter(); final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().commit(); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1, fileContents.getValue()); } } }
for (String file : filesToCopy) { Path source = new Path(file); copyFile(source, tmpPlanFilesPath, source.getName());
@Test public void testCommitAfterPersist() throws Exception { final RecoverableWriter writer = getNewFileSystemWriter(); final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.persist(); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().commit(); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1 + testData2, fileContents.getValue()); } } }
@Test public void testCommitAfterRecovery() throws Exception { final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); final RecoverableWriter initWriter = getNewFileSystemWriter(); final RecoverableWriter.CommitRecoverable recoverable; try (final RecoverableFsDataOutputStream stream = initWriter.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.persist(); stream.persist(); // and write some more data stream.write(testData2.getBytes(StandardCharsets.UTF_8)); recoverable = stream.closeForCommit().getRecoverable(); } final byte[] serializedRecoverable = initWriter.getCommitRecoverableSerializer().serialize(recoverable); // get a new serializer from a new writer to make sure that no pre-initialized state leaks in. final RecoverableWriter newWriter = getNewFileSystemWriter(); final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> deserializer = newWriter.getCommitRecoverableSerializer(); final RecoverableWriter.CommitRecoverable recoveredRecoverable = deserializer.deserialize(deserializer.getVersion(), serializedRecoverable); final RecoverableFsDataOutputStream.Committer committer = newWriter.recoverForCommit(recoveredRecoverable); committer.commitAfterRecovery(); Map<Path, String> files = getFileContentByPath(testDir); Assert.assertEquals(1L, files.size()); for (Map.Entry<Path, String> fileContents : files.entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1 + testData2, fileContents.getValue()); } }
Assert.assertEquals(5, splitsMixed.length); for(FileInputSplit split : splitsMixed) { if(split.getPath().getName().endsWith(".deflate")) {