@Override public void apply() throws Exception { dataset.dropPartition(partitionKey1); } });
public void deleteMatchingPartitionsByTime(long upperLimit) throws IOException { if (upperLimit > 0 && upperLimit < Long.MAX_VALUE) { PartitionFilter filter = PartitionFilter.builder().addRangeCondition(SNAPSHOT_FIELD, null, upperLimit).build(); Set<PartitionDetail> partitions = files.getPartitions(filter); for (PartitionDetail partition : partitions) { files.dropPartition(partition.getPartitionKey()); } } }
@DELETE @Path("partitions/{partition}/subpartitions/{sub-partition}") public void drop(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("partition") String partition, @PathParam("sub-partition") int subPartition) { PartitionKey key = PartitionKey.builder() .addStringField("partition", partition) .addIntField("sub-partition", subPartition) .build(); if (pfs.getPartition(key) == null) { responder.sendString(404, "Partition not found.", Charsets.UTF_8); return; } pfs.dropPartition(key); responder.sendString(200, "Successfully dropped partition", Charsets.UTF_8); } }
@Override public void apply() throws Exception { // drop all existing partitions (2 of which are not consumed) for (PartitionDetail partitionDetail : dataset.getPartitions(PartitionFilter.ALWAYS_MATCH)) { dataset.dropPartition(partitionDetail.getPartitionKey()); } // add 5 new ones for (PartitionKey partitionKey : partitionKeys2) { dataset.getPartitionOutput(partitionKey).addPartition(); } } });
@Override public void apply() throws Exception { Location outputLocation = createPartition(pfs, PARTITION_KEY, "file"); outputLocationRef.set(outputLocation); Assert.assertTrue(outputLocation.exists()); Assert.assertNotNull(pfs.getPartition(PARTITION_KEY)); Assert.assertTrue(pfs.getPartition(PARTITION_KEY).getLocation().exists()); pfs.dropPartition(PARTITION_KEY); Assert.assertFalse(outputLocation.exists()); Assert.assertNull(pfs.getPartition(PARTITION_KEY)); pfs.dropPartition(PARTITION_KEY); } });
@Override public void apply() throws Exception { Assert.assertTrue(pfsBaseLocation.exists()); // attempt to write a new partition - should fail try { pfs.getPartitionOutput(PARTITION_KEY); Assert.fail("External partitioned file set should not allow writing files"); } catch (UnsupportedOperationException e) { // expected } // create an external file and add it as a partition File someFile = new File(absolutePath, "some.file"); OutputStream out = new FileOutputStream(someFile); out.close(); Assert.assertTrue(someFile.exists()); pfs.addPartition(PARTITION_KEY, "some.file"); Assert.assertNotNull(pfs.getPartition(PARTITION_KEY)); Assert.assertTrue(pfs.getPartition(PARTITION_KEY).getLocation().exists()); // now drop the partition and validate the file is still there pfs.dropPartition(PARTITION_KEY); Assert.assertNull(pfs.getPartition(PARTITION_KEY)); Assert.assertTrue(someFile.exists()); } });
pfs.dropPartition(partitionOutput.getPartitionKey()); pfs.dropPartition(partition.getPartitionKey());
@Test public void testRollbackOfPartitionCreateThenDelete() throws Exception { PartitionedFileSet pfs = dsFrameworkUtil.getInstance(pfsInstance); TransactionContext txContext = new TransactionContext(txClient, (TransactionAware) pfs); txContext.start(); Assert.assertNull(pfs.getPartition(PARTITION_KEY)); Location outputLocation = createPartition(pfs, PARTITION_KEY, "file"); Assert.assertNotNull(pfs.getPartition(PARTITION_KEY)); pfs.dropPartition(PARTITION_KEY); txContext.abort(); // the file shouldn't exist because the transaction was aborted (AND because it was dropped at the end of the tx) Assert.assertFalse(outputLocation.exists()); }
pfs.dropPartition(PARTITION_KEY); Assert.assertNull(pfs.getPartition(PARTITION_KEY)); Assert.assertFalse(outputLocation.exists());
pfs.dropPartition(PartitionKey.builder().addField("i", 1).addField("l", 2L).addField("s", 0).build()); Assert.fail("should have thrown exception due to incompatible field"); } catch (IllegalArgumentException e) {