@Override public void setup(List<String> columnNames) throws IOException { this.columnNames = columnNames; this.fs = FileSystemWrapper.get(conf, stats); }
@Override public FileSystem getFileSystem(Configuration conf) throws IOException { // Do not return the original filesystem as-is because: // - the path might have been canonicalized and might point to a different URI (viewFS) // - this method might be called from a different security context (and the filesystem // should reflect this) // Instead recreate a new wrapper (but keep the stats context) return FileSystemWrapper.get(this.toUri(), conf, fs.getOperatorStats()); } }
private boolean hasChanged() throws IOException { final JobConf jobConf = new JobConf(hiveConf); for (Prop prop : HiveReaderProtoUtil.getPartitionProperties(tableXattr, updateKey.getPartitionId())) { jobConf.set(prop.getKey(), prop.getValue()); } Preconditions.checkArgument(updateKey.getCachedEntitiesCount() > 0, "hive partition update key should contain at least one path"); // create filesystem based on the first path which is root of the partition directory. final FileSystemWrapper fs = FileSystemWrapper.get(new Path(updateKey.getPartitionRootDir()), jobConf); for (FileSystemCachedEntity cachedEntity : updateKey.getCachedEntitiesList()) { final Path cachedEntityPath; if (cachedEntity.getPath() == null || cachedEntity.getPath().isEmpty()) { cachedEntityPath = new Path(updateKey.getPartitionRootDir()); } else { cachedEntityPath = new Path(updateKey.getPartitionRootDir(), cachedEntity.getPath()); } if (fs.exists(cachedEntityPath)) { final FileStatus fileStatus = fs.getFileStatus(cachedEntityPath); if (cachedEntity.getLastModificationTime() < fileStatus.getModificationTime()) { return true; } } else { return true; } } return false; } }
@Override public void setup() throws IOException { this.fs = FileSystemWrapper.get(conf, stats); }
private FileSystemPartitionUpdateKey getFSBasedUpdateKey(String partitionDir, JobConf job, boolean isRecursive, int partitionId) throws IOException { final List<FileSystemCachedEntity> cachedEntities = Lists.newArrayList(); final Path rootLocation = new Path(partitionDir); final FileSystemWrapper fs = FileSystemWrapper.get(rootLocation, job);
@Override public FileSystemWrapper run() throws Exception { logger.trace("Creating FileSystemWrapper for proxy user: " + UserGroupInformation.getCurrentUser()); return FileSystemWrapper.get(path, fsConf); } });
private static boolean inputPathExists(StorageDescriptor sd, JobConf job) throws IOException { final Path path = new Path(sd.getLocation()); final FileSystem fs = FileSystemWrapper.get(path, job); if (fs.exists(path)) { return true; } return false; }
@Override public void setup() throws IOException { this.fs = FileSystemWrapper.get(conf, context.getStats()); this.batchSchema = incoming.getSchema(); newSchema(); }
@Override public void setup(VectorAccessible incoming, OutputEntryListener outputEntryListener, WriteStatsListener writeStatsListener) throws IOException { Preconditions.checkArgument(incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE, "SelectionVector remover is not supported."); this.incoming = incoming; this.outputEntryListener = outputEntryListener; this.writeStatsListener = writeStatsListener; this.fs = FileSystemWrapper.get(location, writerConfig.getFsConf(), stats); this.currentFile = fs.canonicalizePath(new Path(location, String.format("%s_%d.%s", prefix, nextFileIndex, extension))); this.relativePath = currentFile.getName(); this.currentFileOutputStream = fs.create(currentFile); listOfFilesCreated.add(currentFile); // write magic word bytes currentFileOutputStream.write(MAGIC_STRING.getBytes()); for(final VectorWrapper<? extends ValueVector> vw : incoming) { Preconditions.checkArgument(!vw.isHyper(), "Writing hyper vectors to arrow format is not supported."); footerBuilder.addField(TypeHelper.getMetadata(vw.getValueVector())); } nextFileIndex++; recordCount = 0; }
public VectorContainer loadDictionary(String fieldName) throws IOException, ExecutionSetupException { final StoragePluginId id = config.getDictionaryEncodedFields().get(fieldName).getStoragePluginId(); final StoragePlugin storagePlugin = config.getCatalogService().getSource(id); if (storagePlugin instanceof FileSystemPlugin) { final FileSystemPlugin fsPlugin = (FileSystemPlugin) storagePlugin; final FileSystem fs = FileSystemWrapper.get(fsPlugin.getFsConf()); return ParquetFormatPlugin.loadDictionary(fs, new Path(config.getDictionaryEncodedFields().get(fieldName).getDictionaryPath()), context.getAllocator()); } else { throw new ExecutionSetupException(format("Storage plugin %s is not a filesystem plugin", id.getName())); } }
public FileSystemWrapper getFilesystemAndCreatePaths(String hostname) throws IOException { FileSystemWrapper fs = FileSystemWrapper.get(uri.get(), new Configuration()); fs.mkdirs(getPath(), HomeFileSystemStoragePlugin.DEFAULT_PERMISSIONS); fs.mkdirs(getInnerUploads(), HomeFileSystemStoragePlugin.DEFAULT_PERMISSIONS); if(hostname == null) { hostname = this.hostname; } if(hostname != null) { fs.mkdirs(getStagingPath(hostname), HomeFileSystemStoragePlugin.DEFAULT_PERMISSIONS); fs.deleteOnExit(getStagingPath(hostname)); } return fs; }
@Test public void testFileNotFound() { FileSplit split = mock(FileSplit.class); when(split.getPath()).thenReturn(new Path("/notExist/notExitFile")); TextParsingSettings settings = mock(TextParsingSettings.class); when(settings.isHeaderExtractionEnabled()).thenReturn(true); SchemaPath column = mock(SchemaPath.class); List<SchemaPath> columns = new ArrayList<>(1); columns.add(column); SabotContext context = mock(SabotContext.class); BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); when(context.getAllocator()).thenReturn(allocator); Path path = new Path("/notExist"); try (BufferAllocator sampleAllocator = context.getAllocator().newChildAllocator("sample-alloc", 0, Long.MAX_VALUE); OperatorContextImpl operatorContext = new OperatorContextImpl(context.getConfig(), sampleAllocator, context.getOptionManager(), 1000); FileSystemWrapper dfs = FileSystemWrapper.get(path, new Configuration()); SampleMutator mutator = new SampleMutator(sampleAllocator); CompliantTextRecordReader reader = new CompliantTextRecordReader(split, dfs, operatorContext, settings, columns); ){ reader.setup(mutator); } catch (Exception e) { // java.io.FileNotFoundException is expected, but memory leak is not expected. assertTrue(e.getCause() instanceof FileNotFoundException); } allocator.close(); } }
fs = FileSystemWrapper.get(finalPath, jobConf, oContext.getStats()); } catch(IOException e) { throw new ExecutionSetupException(String.format("Failed to create FileSystem: %s", e.getMessage()), e);