private List<String> listPartitionsAfterCheckpoint(final Optional<StringValue> checkpoint) throws IOException { final List<String> partitions = getExistingPartitions(); if (checkpoint.isPresent()) { return partitions.stream() .filter(partition -> partition.compareTo(checkpoint.get().getValue()) > 0) .collect(Collectors.toList()); } else { return partitions; } }
@Test public void testGetExistingPartitions() throws IOException { final Path partition0Path = new Path(RAW_DATA_PATH, PARTITION0); final Path partition1Path = new Path(RAW_DATA_PATH, PARTITION1); final Path partition2Path = new Path(RAW_DATA_PATH, PARTITION2); this.fileSystem.create(new Path(partition0Path, FILE1)); this.fileSystem.create(new Path(partition1Path, FILE1)); this.fileSystem.create(new Path(partition2Path, FILE1)); final HDFSPartitionManager pm = new HDFSPartitionManager(JOBNAME, HDFSTestConstants.BASE_METADATA_PATH, RAW_DATA_PATH, this.fileSystem); Assert.assertEquals(3, pm.getExistingPartitions().size()); for (int i = 0; i < pm.getExistingPartitions().size(); i++) { Assert.assertEquals("partition" + i, pm.getExistingPartitions().get(i)); } }
@Test public void testGetExistingPartitionsOnlyFilesExist() throws IOException { final Path partition0File = new Path(RAW_DATA_PATH, PARTITION0); this.fileSystem.create(partition0File); final HDFSPartitionManager pm = new HDFSPartitionManager(JOBNAME, HDFSTestConstants.BASE_METADATA_PATH, RAW_DATA_PATH, this.fileSystem); final HDFSMetadataManager metadataManager = new HDFSMetadataManager(this.fileSystem, new Path(HDFSTestConstants.BASE_METADATA_PATH, JOBNAME).toString(), new AtomicBoolean(true)); Assert.assertEquals(1, pm.getExistingPartitions().size()); Assert.assertEquals(RAW_DATA_PATH, pm.getNextPartition(getLatestCheckpoint(metadataManager)).get()); }