@Override public void cancel() { Closeables.closeQuietly(writer); Locations.deleteQuietly(Locations.getParent(eventFile), true); }
/** * Gets the stream name given a stream's base directory. * @param streamBaseLocation the location of the stream's directory * @return name of the stream associated with the location */ public static String getStreamNameFromLocation(Location streamBaseLocation) { // streamBaseLocation = /.../<namespace>/streams/<streamName>, // or /customLocation/streams/<streamname> // as constructed by FileStreamAdmin#getStreamConfigLocation Location streamsDir = Locations.getParent(streamBaseLocation); Preconditions.checkNotNull(streamsDir, "Streams directory of stream base location %s was null.", streamBaseLocation); return streamBaseLocation.getName(); }
private void assertNamespaceHomeExists(NamespaceId namespaceId) throws IOException { Location namespaceHomeLocation = Locations.getParent(getStreamBaseLocation(namespaceId)); Preconditions.checkArgument(namespaceHomeLocation != null && namespaceHomeLocation.exists(), "Home directory %s for namespace %s not found", namespaceHomeLocation, namespaceId); }
@Override public void close() throws IOException { try { writer.flush(); streamWriter.appendFile(streamConfig.getStreamId(), eventFile, indexFile, eventCount, writer); } catch (NotFoundException e) { throw Throwables.propagate(e); } finally { Locations.deleteQuietly(Locations.getParent(eventFile), true); } }
/** * Renames the old location to new location if old location exists and the new one does not * * @param oldLocation the old {@link Location} * @param newLocation the new {@link Location} * @return new location if and only if the file or directory is successfully moved; null otherwise. * @throws IOException */ @Nullable protected Location renameLocation(Location oldLocation, Location newLocation) throws IOException { // if the newLocation does not exists or the oldLocation does we try to rename. If either one of them is false then // the underlying call to renameTo will throw IOException which we re-throw. if (!newLocation.exists() && oldLocation.exists()) { Locations.getParent(newLocation).mkdirs(); try { return oldLocation.renameTo(newLocation); } catch (IOException ioe) { newLocation.delete(); LOG.warn("Failed to rename {} to {}", oldLocation, newLocation); throw ioe; } } else { LOG.debug("Failed to perform rename. Either the new location {} already exists or old location {} " + "does not exist.", newLocation, oldLocation); return null; } } }
private StreamPositionTransformFileReader(StreamFileOffset offset) throws IOException { this.reader = StreamDataFileReader.createWithOffset(Locations.newInputSupplier(offset.getEventLocation()), Locations.newInputSupplier(offset.getIndexLocation()), offset.getOffset()); this.offset = new StreamFileOffset(offset); this.partitionLocation = Locations.getParent(offset.getEventLocation()); LOG.trace("Stream reader created for {}", offset.getEventLocation()); }
@Test public void testRootParentFile() { Configuration conf = new Configuration(); conf.set("fs.defaultFS", "file:///"); LocationFactory locationFactory = new FileContextLocationFactory(conf, "abc"); Location location = locationFactory.create("def"); Assert.assertEquals("def", location.getName()); location = Locations.getParent(location); Assert.assertNotNull(location); Assert.assertEquals("abc", location.getName()); location = Locations.getParent(location); Assert.assertNotNull(location); Assert.assertTrue(Locations.isRoot(location)); Assert.assertTrue(location.getName().isEmpty()); Assert.assertNull(Locations.getParent(location)); }
@Test public void testParentHDFS() { Configuration conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://1.2.3.4:8020"); LocationFactory locationFactory = new FileContextLocationFactory(conf, "abc"); Location location = locationFactory.create("def"); Assert.assertEquals("def", location.getName()); location = Locations.getParent(location); Assert.assertNotNull(location); Assert.assertEquals("abc", location.getName()); location = Locations.getParent(location); Assert.assertNotNull(location); Assert.assertTrue(Locations.isRoot(location)); Assert.assertTrue(location.getName().isEmpty()); Assert.assertNull(Locations.getParent(location)); }
@Test public void testRootParentLocal() throws IOException { LocationFactory locationFactory = new LocalLocationFactory(new File(File.separator)); Location location = locationFactory.create("abc"); location = location.append("def"); Assert.assertEquals("def", location.getName()); location = Locations.getParent(location); Assert.assertNotNull(location); Assert.assertEquals("abc", location.getName()); location = Locations.getParent(location); Assert.assertNotNull(location); Assert.assertTrue(Locations.isRoot(location)); Assert.assertTrue(location.getName().isEmpty()); Assert.assertNull(Locations.getParent(location)); }
if (partitionLocation.exists()) { Location dstLocation = getQuarantineLocation().append(partition.getRelativePath()); Location dstParent = Locations.getParent(dstLocation);
@Test public void testWriteRead() throws IOException, DatasetManagementException { FileSet fileSet1 = createFileset(testFileSetInstance1); FileSet fileSet2 = createFileset(testFileSetInstance2); Location fileSet1Output = fileSet1.getOutputLocation(); Location fileSet2Output = fileSet2.getOutputLocation(); Location fileSet1NsDir = Locations.getParent(Locations.getParent(Locations.getParent(fileSet1Output))); Location fileSet2NsDir = Locations.getParent(Locations.getParent(Locations.getParent(fileSet2Output))); Assert.assertNotNull(fileSet1NsDir); Assert.assertNotNull(fileSet2NsDir); Assert.assertEquals(fileSet1NsDir.getName(), DatasetFrameworkTestUtil.NAMESPACE_ID.getNamespace()); Assert.assertEquals(fileSet2NsDir.getName(), OTHER_NAMESPACE.getNamespace()); Assert.assertNotEquals(fileSet1.getInputLocations().get(0).toURI().getPath(), fileSet2.getInputLocations().get(0).toURI().getPath()); Assert.assertNotEquals(fileSet1Output.toURI().getPath(), fileSet2Output.toURI().getPath()); try (OutputStream out = fileSet1.getOutputLocation().getOutputStream()) { out.write(42); } try (OutputStream out = fileSet2.getOutputLocation().getOutputStream()) { out.write(54); } try (InputStream in = fileSet1.getInputLocations().get(0).getInputStream()) { Assert.assertEquals(42, in.read()); } try (InputStream in = fileSet2.getInputLocations().get(0).getInputStream()) { Assert.assertEquals(54, in.read()); } }
while (ancestor != null && (Locations.isRoot(ancestor) || !ancestor.exists())) { firstDirToCreate = ancestor; ancestor = Locations.getParent(ancestor);
if (partitionLocation.exists()) { Location dstLocation = getQuarantineLocation().append(partition.getRelativePath()); Location dstParent = Locations.getParent(dstLocation);
while (ancestor != null && (Locations.isRoot(ancestor) || !ancestor.exists())) { firstDirToCreate = ancestor; ancestor = Locations.getParent(ancestor);
StreamUtils.createPartitionLocation(Locations.getParent(reader.getPartitionLocation()), offset.getPartitionEnd(), streamConfig.getPartitionDuration()), offset.getNamePrefix(), 0, StreamFileType.EVENT);
@Test public void testDefaultBasePath() throws Exception { DatasetId id = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("testDefaultPath"); dsFrameworkUtil.createInstance("partitionedFileSet", id, PartitionedFileSetProperties.builder() .setPartitioning(PARTITIONING_1) .build()); PartitionedFileSet pfs = dsFrameworkUtil.getInstance(id); Location baseLocation = pfs.getEmbeddedFileSet().getBaseLocation(); Assert.assertEquals(baseLocation.getName(), id.getDataset()); Assert.assertTrue(baseLocation.exists()); Assert.assertTrue(baseLocation.isDirectory()); DatasetId fid = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("testDefaultPathFileSet"); dsFrameworkUtil.createInstance("fileSet", fid, FileSetProperties.builder().build()); FileSet fs = dsFrameworkUtil.getInstance(fid); Location fsBaseLocation = fs.getBaseLocation(); Assert.assertEquals(Locations.getParent(baseLocation), Locations.getParent(fsBaseLocation)); dsFrameworkUtil.deleteInstance(fid); dsFrameworkUtil.deleteInstance(id); Assert.assertFalse(baseLocation.exists()); }
Assert.assertEquals(group, base.getGroup()); Assert.assertEquals(fsPermissions, base.getPermissions()); Location parent = Locations.getParent(base); Assert.assertNotNull(parent); Assert.assertEquals(group, parent.getGroup()); Assert.assertEquals(fsPermissions, parent.getPermissions()); parent = Locations.getParent(parent); Assert.assertNotNull(parent); Assert.assertEquals(group, parent.getGroup()); Assert.assertEquals(fsPermissions, parent.getPermissions()); Location nsRoot = Locations.getParent(parent); Assert.assertNotNull(nsRoot); Assert.assertNotEquals(fsPermissions, nsRoot.getPermissions());