/** * The mapreduce version of the post processor writes different clusters into different part files. This * method reads the part files and moves them into directories named after their clusterIds. * * @param conf The hadoop configuration. * @param output The post processed data would be stored at this path. */ private static void movePartFilesToRespectiveDirectories(Configuration conf, Path output) throws IOException { FileSystem fileSystem = output.getFileSystem(conf); for (FileStatus fileStatus : fileSystem.listStatus(output, PathFilters.partFilter())) { SequenceFileIterator<Writable, Writable> it = new SequenceFileIterator<Writable, Writable>(fileStatus.getPath(), true, conf); if (it.hasNext()) { renameFile(it.next().getFirst(), fileStatus, conf); } it.close(); } }
/** * The mapreduce version of the post processor writes different clusters into different part files. This * method reads the part files and moves them into directories named after their clusterIds. * * @param conf The hadoop configuration. * @param output The post processed data would be stored at this path. */ private static void movePartFilesToRespectiveDirectories(Configuration conf, Path output) throws IOException { FileSystem fileSystem = output.getFileSystem(conf); for (FileStatus fileStatus : fileSystem.listStatus(output, PathFilters.partFilter())) { SequenceFileIterator<Writable, Writable> it = new SequenceFileIterator<Writable, Writable>(fileStatus.getPath(), true, conf); if (it.hasNext()) { renameFile(it.next().getFirst(), fileStatus, conf); } it.close(); } }
/** * The mapreduce version of the post processor writes different clusters into different part files. This * method reads the part files and moves them into directories named after their clusterIds. * * @param conf The hadoop configuration. * @param output The post processed data would be stored at this path. */ private static void movePartFilesToRespectiveDirectories(Configuration conf, Path output) throws IOException { FileSystem fileSystem = output.getFileSystem(conf); for (FileStatus fileStatus : fileSystem.listStatus(output, PathFilters.partFilter())) { SequenceFileIterator<Writable, Writable> it = new SequenceFileIterator<>(fileStatus.getPath(), true, conf); if (it.hasNext()) { renameFile(it.next().getFirst(), fileStatus, conf); } it.close(); } }