private Long getLatestSnapshot() throws IOException { Location stateFile = files.getEmbeddedFileSet().getBaseLocation().append(STATE_FILE_NAME); if (!stateFile.exists()) { return null; } try (InputStreamReader reader = new InputStreamReader(stateFile.getInputStream(), Charsets.UTF_8)) { String val = CharStreams.toString(reader); return Long.valueOf(val); } }
private Location lock() throws IOException, InterruptedException { // create a lock file in case there is somebody updating the latest snapshot Location lockFile = files.getEmbeddedFileSet().getBaseLocation().append("lock"); int retries = 0; int maxRetries = 20; while (!lockFile.createNew()) { if (retries > maxRetries) { throw new IOException("Failed to create lock file. If there is a file named 'lock' in the " + "base path, but there is nobody updating the latest snapshot, please delete the 'lock' file."); } TimeUnit.SECONDS.sleep(1); retries++; } return lockFile; } }
public void onSuccess(long snapshotTime) throws IOException, InterruptedException { Location lock = lock(); try { // update state file that contains the latest snapshot Long latestSnapshot = getLatestSnapshot(); if (latestSnapshot == null || snapshotTime > latestSnapshot) { Location stateFile = files.getEmbeddedFileSet().getBaseLocation().append(STATE_FILE_NAME); stateFile.delete(); try (OutputStream outputStream = stateFile.getOutputStream()) { outputStream.write(String.valueOf(snapshotTime).getBytes(Charsets.UTF_8)); } } } finally { lock.delete(); } }
private void validateFiles(String dataset, Location expectedExisting) throws Exception { DataSetManager<PartitionedFileSet> pfs = getDataset(testSpace.dataset(dataset)); Location base = pfs.get().getEmbeddedFileSet().getBaseLocation(); validateFiles(base, expectedExisting); }
if (dataset instanceof PartitionedFileSet) { partitioning = ((PartitionedFileSet) dataset).getPartitioning(); baseLocation = ((PartitionedFileSet) dataset).getEmbeddedFileSet().getBaseLocation(); } else { baseLocation = ((FileSet) dataset).getBaseLocation();
if (dataset instanceof PartitionedFileSet) { partitioning = ((PartitionedFileSet) dataset).getPartitioning(); baseLocation = ((PartitionedFileSet) dataset).getEmbeddedFileSet().getBaseLocation(); } else { baseLocation = ((FileSet) dataset).getBaseLocation();
location2 = pfs.getEmbeddedFileSet().getLocation(path2); try (Writer writer = new OutputStreamWriter(location2.append("file").getOutputStream())) { writer.write("2,2\n"); String basePath = pfs.getEmbeddedFileSet().getBaseLocation().toURI().getPath(); String absPath3 = location3.toURI().getPath(); Assert.assertTrue(absPath3.startsWith(basePath));
@Test public void testPermissions() throws Exception { // validate that the fileset permissions and group were applied to the embedded fileset (just sanity test) PartitionedFileSet pfs = dsFrameworkUtil.getInstance(pfsInstance); Location loc = pfs.getEmbeddedFileSet().getLocation("some/random/path"); loc.getOutputStream().close(); Assert.assertEquals(fsPermissions, loc.getPermissions()); Assert.assertEquals(group, loc.getGroup()); Map<String, String> props = dsFrameworkUtil.getSpec(pfsInstance).getSpecification("partitions").getProperties(); Assert.assertEquals(tablePermissions, TableProperties.getTablePermissions(props)); }
@Before public void before() throws Exception { dsFrameworkUtil.createInstance("partitionedFileSet", pfsInstance, PartitionedFileSetProperties.builder() .setPartitioning(PARTITIONING_1) .setBasePath("testDir") .build()); pfsBaseLocation = ((PartitionedFileSet) dsFrameworkUtil.getInstance(pfsInstance)) .getEmbeddedFileSet().getBaseLocation(); Assert.assertTrue(pfsBaseLocation.exists()); }
Location loc = pfs.get().getEmbeddedFileSet().getLocation("some/path"); OutputStream os = loc.append("part1").getOutputStream(); try (Writer writer = new OutputStreamWriter(os)) {
@Before public void before() throws Exception { txClient = new InMemoryTxSystemClient(dsFrameworkUtil.getTxManager()); dsFrameworkUtil.createInstance("partitionedFileSet", pfsInstance, PartitionedFileSetProperties.builder() .setPartitioning(PARTITIONING_1) .setTablePermissions(tablePermissions) .setBasePath("testDir") .setFilePermissions(fsPermissions) .setFileGroup(group) .build()); pfsBaseLocation = ((PartitionedFileSet) dsFrameworkUtil.getInstance(pfsInstance)) .getEmbeddedFileSet().getBaseLocation(); Assert.assertTrue(pfsBaseLocation.exists()); }
@Test public void testRollbackOnJobFailure() throws Exception { // tests the logic of #onFailure method Map<String, String> args = new HashMap<>(); FileSetArguments.setOutputPath(args, "custom/output/path"); PartitionedFileSetArguments.setOutputPartitionKey(args, PARTITION_KEY); PartitionedFileSet pfs = dsFrameworkUtil.getInstance(pfsInstance, args); TransactionContext txContext = new TransactionContext(txClient, (TransactionAware) pfs); txContext.start(); Location outputLocation = pfs.getEmbeddedFileSet().getOutputLocation(); Assert.assertFalse(outputLocation.exists()); outputLocation.mkdirs(); Assert.assertTrue(outputLocation.exists()); ((PartitionedFileSetDataset) pfs).onFailure(); txContext.abort(); // because the previous transaction aborted, the partition as well as the directory for it will not exist txContext.start(); Assert.assertNull(pfs.getPartition(PARTITION_KEY)); Assert.assertFalse(outputLocation.exists()); txContext.finish(); }
@Test public void testDefaultBasePath() throws Exception { DatasetId id = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("testDefaultPath"); dsFrameworkUtil.createInstance("partitionedFileSet", id, PartitionedFileSetProperties.builder() .setPartitioning(PARTITIONING_1) .build()); PartitionedFileSet pfs = dsFrameworkUtil.getInstance(id); Location baseLocation = pfs.getEmbeddedFileSet().getBaseLocation(); Assert.assertEquals(baseLocation.getName(), id.getDataset()); Assert.assertTrue(baseLocation.exists()); Assert.assertTrue(baseLocation.isDirectory()); DatasetId fid = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("testDefaultPathFileSet"); dsFrameworkUtil.createInstance("fileSet", fid, FileSetProperties.builder().build()); FileSet fs = dsFrameworkUtil.getInstance(fid); Location fsBaseLocation = fs.getBaseLocation(); Assert.assertEquals(Locations.getParent(baseLocation), Locations.getParent(fsBaseLocation)); dsFrameworkUtil.deleteInstance(fid); dsFrameworkUtil.deleteInstance(id); Assert.assertFalse(baseLocation.exists()); }