@Nullable public Location getLocation() throws IOException, InterruptedException { Location lock = lock(); try { PartitionDetail partitionDetail = getLatestPartition(); if (partitionDetail == null) { return null; } return partitionDetail.getLocation(); } finally { lock.delete(); } }
@GET @Path("partitions/{partition}/subpartitions/{sub-partition}") public void read(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("partition") String partition, @PathParam("sub-partition") int subPartition) { PartitionDetail partitionDetail = pfs.getPartition(PartitionKey.builder() .addStringField("partition", partition) .addIntField("sub-partition", subPartition) .build()); if (partitionDetail == null) { responder.sendString(404, "Partition not found.", Charsets.UTF_8); return; } try { responder.send(200, partitionDetail.getLocation().append("file"), "text/plain"); } catch (IOException e) { responder.sendError(400, String.format("Unable to read path '%s'", partitionDetail.getRelativePath())); } }
partitionsTable.delete(rowKey); if (!isExternal) { Location partitionLocation = partition.getLocation(); try { if (partitionLocation.exists()) {
partitionsTable.delete(rowKey); if (!isExternal) { Location partitionLocation = partition.getLocation(); try { if (partitionLocation.exists()) {
@Override public void run(DatasetContext context) throws Exception { Map<String, Long> wordCounts = new HashMap<>(); for (PartitionDetail partition : partitions) { ByteBuffer content; Location location = partition.getLocation(); content = ByteBuffer.wrap(ByteStreams.toByteArray(location.getInputStream())); String string = Bytes.toString(Bytes.toBytes(content)); for (String token : string.split(" ")) { Long count = Objects.firstNonNull(wordCounts.get(token), 0L); wordCounts.put(token, count + 1); } } IncrementingKeyValueTable counts = context.getDataset("counts"); for (Map.Entry<String, Long> entry : wordCounts.entrySet()) { counts.write(Bytes.toBytes(entry.getKey()), entry.getValue()); } PartitionedFileSet outputLines = context.getDataset("outputLines"); PartitionKey partitionKey = PartitionKey.builder().addLongField("time", System.currentTimeMillis()).build(); PartitionOutput outputPartition = outputLines.getPartitionOutput(partitionKey); Location partitionDir = outputPartition.getLocation(); partitionDir.mkdirs(); Location outputLocation = partitionDir.append("file"); outputLocation.createNew(); try (OutputStream outputStream = outputLocation.getOutputStream()) { outputStream.write(Bytes.toBytes(Joiner.on("\n").join(wordCounts.values()))); } outputPartition.addPartition(); } });
Assert.assertNotNull("Output partition is null while for running without custom dataset arguments", partition); validateFileOutput(partition.getLocation()); validateFileOutput(customPartition.getLocation());
@Override public void apply() throws Exception { Assert.assertTrue(pfsBaseLocation.exists()); // attempt to write a new partition - should fail try { pfs.getPartitionOutput(PARTITION_KEY); Assert.fail("External partitioned file set should not allow writing files"); } catch (UnsupportedOperationException e) { // expected } // create an external file and add it as a partition File someFile = new File(absolutePath, "some.file"); OutputStream out = new FileOutputStream(someFile); out.close(); Assert.assertTrue(someFile.exists()); pfs.addPartition(PARTITION_KEY, "some.file"); Assert.assertNotNull(pfs.getPartition(PARTITION_KEY)); Assert.assertTrue(pfs.getPartition(PARTITION_KEY).getLocation().exists()); // now drop the partition and validate the file is still there pfs.dropPartition(PARTITION_KEY); Assert.assertNull(pfs.getPartition(PARTITION_KEY)); Assert.assertTrue(someFile.exists()); } });
List<Location> locations = partition.getLocation().list(); Assert.assertEquals(1, locations.size()); Assert.assertArrayEquals(content, ByteStreams.toByteArray(Locations.newInputSupplier(locations.get(0))));
@Override public void apply() throws Exception { Location outputLocation = createPartition(pfs, PARTITION_KEY, "file"); outputLocationRef.set(outputLocation); Assert.assertTrue(outputLocation.exists()); Assert.assertNotNull(pfs.getPartition(PARTITION_KEY)); Assert.assertTrue(pfs.getPartition(PARTITION_KEY).getLocation().exists()); pfs.dropPartition(PARTITION_KEY); Assert.assertFalse(outputLocation.exists()); Assert.assertNull(pfs.getPartition(PARTITION_KEY)); pfs.dropPartition(PARTITION_KEY); } });
PartitionDetail partition = pfs.getPartition(outputKey); Assert.assertNotNull(partition); validateFileOutput(partition.getLocation());
@Override public void run() { try { PartitionDetail detail1 = pfs.getPartition(KEY_1); Assert.assertNotNull(detail1); Assert.assertEquals(location1, detail1.getLocation()); Assert.assertTrue(location1.exists()); Assert.assertTrue(location1.append("file").exists()); PartitionDetail detail2 = pfs.getPartition(KEY_2); Assert.assertNotNull(detail2); Assert.assertEquals(location2, detail2.getLocation()); Assert.assertTrue(location2.exists()); Assert.assertTrue(location2.append("file").exists()); PartitionDetail detail3 = pfs.getPartition(KEY_4); Assert.assertNull(detail3); Assert.assertTrue(location3.exists()); Assert.assertTrue(location3.append("file").exists()); } catch (Exception e) { throw Throwables.propagate(e); } } });
@Test public void testRollbackOnTransactionAbort() throws Exception { PartitionedFileSet pfs = dsFrameworkUtil.getInstance(pfsInstance); TransactionContext txContext = new TransactionContext(txClient, (TransactionAware) pfs); txContext.start(); Location outputLocation = createPartition(pfs, PARTITION_KEY, "file");; Assert.assertNotNull(pfs.getPartition(PARTITION_KEY)); Assert.assertTrue(pfs.getPartition(PARTITION_KEY).getLocation().exists()); txContext.abort(); // because the previous transaction aborted, the partition as well as the file will not exist txContext.start(); Assert.assertNull(pfs.getPartition(PARTITION_KEY)); Assert.assertFalse(outputLocation.exists()); txContext.finish(); }
Assert.assertTrue(pfs.getPartition(PARTITION_KEY).getLocation().exists()); txContext.finish();
Assert.assertFalse(pfs.getPartition(PARTITION_KEY).getLocation().append("file2").exists()); txContext.finish();