/** * Deletes any new tmp files written during the current commit, into the partition */ public static void cleanupTmpFilesFromCurrentCommit(HoodieWriteConfig config, String commitTime, String partitionPath, int taskPartitionId, HoodieTable hoodieTable) { FileSystem fs = hoodieTable.getMetaClient().getFs(); try { FileStatus[] prevFailedFiles = fs.globStatus(new Path(String .format("%s/%s/%s", config.getBasePath(), partitionPath, FSUtils.maskWithoutFileId(commitTime, taskPartitionId)))); if (prevFailedFiles != null) { logger.info( "Deleting " + prevFailedFiles.length + " files generated by previous failed attempts."); for (FileStatus status : prevFailedFiles) { fs.delete(status.getPath(), false); } } } catch (IOException e) { throw new HoodieIOException("Failed to cleanup Temp files from commit " + commitTime, e); } }
/** * Deletes any new tmp files written during the current commit, into the partition */ public static void cleanupTmpFilesFromCurrentCommit(HoodieWriteConfig config, String commitTime, String partitionPath, int taskPartitionId, HoodieTable hoodieTable) { FileSystem fs = hoodieTable.getMetaClient().getFs(); try { FileStatus[] prevFailedFiles = fs.globStatus(new Path(String .format("%s/%s/%s", config.getBasePath(), partitionPath, FSUtils.maskWithoutFileId(commitTime, taskPartitionId)))); if (prevFailedFiles != null) { logger.info( "Deleting " + prevFailedFiles.length + " files generated by previous failed attempts."); for (FileStatus status : prevFailedFiles) { fs.delete(status.getPath(), false); } } } catch (IOException e) { throw new HoodieIOException("Failed to cleanup Temp files from commit " + commitTime, e); } }
@Test public void testMaskFileName() { String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); int taskPartitionId = 2; assertTrue(FSUtils.maskWithoutFileId(commitTime, taskPartitionId) .equals("*_" + taskPartitionId + "_" + commitTime + ".parquet")); }