return ((LocalFileSystem) fs).getRaw();
protected void open(Path dstPath, CompressionCodec codeC, CompressionType compType, Configuration conf, FileSystem hdfs) throws IOException { if (useRawLocalFileSystem) { if (hdfs instanceof LocalFileSystem) { hdfs = ((LocalFileSystem)hdfs).getRaw(); } else { logger.warn("useRawLocalFileSystem is set to true but file system " + "is not of type LocalFileSystem: " + hdfs.getClass().getName()); } } if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) { outStream = hdfs.append(dstPath); } else { outStream = hdfs.create(dstPath); } writer = SequenceFile.createWriter(conf, outStream, serializer.getKeyClass(), serializer.getValueClass(), compType, codeC); registerCurrentStream(outStream, hdfs, dstPath); }
protected void doOpen(Configuration conf, Path dstPath, FileSystem hdfs) throws IOException { if (useRawLocalFileSystem) { if (hdfs instanceof LocalFileSystem) { hdfs = ((LocalFileSystem)hdfs).getRaw(); } else { logger.warn("useRawLocalFileSystem is set to true but file system " +
if (useRawLocalFileSystem) { if (hdfs instanceof LocalFileSystem) { hdfs = ((LocalFileSystem)hdfs).getRaw(); } else { logger.warn("useRawLocalFileSystem is set to true but file system " +
FileSystem localFs = FileSystem.getLocal(conf).getRaw(); Path mapXml = new Path(workDir, "map.xml"); localFs.delete(mapXml, true);
Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf).getRaw(); ObjectInspector inspector; synchronized (TestOrcFile.class) {
@VisibleForTesting TextWriter createWriter(Path file, CompressionCodec codec, String delim, Configuration cfg) throws IOException { FileSystem fs = file.getFileSystem(cfg); if (fs instanceof LocalFileSystem) { fs = ((LocalFileSystem)fs).getRaw(); } OutputStream tmp = fs.create(file); java.io.Writer out = new BufferedWriter(new OutputStreamWriter( (null == codec) ? tmp : codec.createOutputStream(tmp), "UTF-8")); return new TextWriter(out, delim); }
Configuration conf = new Configuration(); OrcOutputFormat of = new OrcOutputFormat(); FileSystem fs = FileSystem.getLocal(conf).getRaw(); Path root = new Path(tmpDir, "testRecordReaderIncompleteDelta").makeQualified(fs); fs.delete(root, true);
@VisibleForTesting TextReader createReader(Path file, String delim, Configuration cfg, String blockPoolID) throws IOException { FileSystem fs = file.getFileSystem(cfg); if (fs instanceof LocalFileSystem) { fs = ((LocalFileSystem)fs).getRaw(); } CompressionCodecFactory factory = new CompressionCodecFactory(cfg); CompressionCodec codec = factory.getCodec(file); String filename = fileNameFromBlockPoolID(blockPoolID); if (codec != null) { filename = filename + codec.getDefaultExtension(); } Path bpidFilePath = new Path(file.getParent(), filename); return new TextReader(fs, bpidFilePath, codec, delim); }
@Before public void setUp() throws IOException { conf = new JobConf(); fs = FileSystem.getLocal(conf).getRaw(); p = new Path(System.getProperty("test.build.data", "/tmp"), "cache").makeQualified(fs.getUri(), fs.getWorkingDirectory()); }
Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf).getRaw(); ObjectInspector inspector; synchronized (TestOrcFile.class) {
Path shuffledToDisk = OnDiskMapOutput.getTempPath(onDiskMapOutputPath, fetcher); fs = FileSystem.getLocal(job).getRaw(); IFileWrappedMapOutput<Text,Text> odmo = new OnDiskMapOutput<Text,Text>(map1ID, mm, 100L, job, fetcher, true,
public static FileSystem getFileSystem(URI uri) throws IOException { final Configuration config = new Configuration(); FileSystem fs = FileSystem.get(uri, config); if (fs instanceof LocalFileSystem) fs = ((LocalFileSystem) fs).getRaw(); return fs; }
/** * @param uri * @return the file system of a given path (HDFS or Local usually) * @throws IOException */ public static FileSystem getFileSystem(URI uri) throws IOException { Configuration config = new Configuration(); FileSystem fs = FileSystem.get(uri, config); if (fs instanceof LocalFileSystem) fs = ((LocalFileSystem)fs).getRaw(); return fs; }
static FileSystem getFileSystem(URI uri) throws IOException { final Configuration config = new Configuration(); FileSystem fs = FileSystem.get(uri, config); if (fs instanceof LocalFileSystem) fs = ((LocalFileSystem) fs).getRaw(); return fs; }
public static FileSystem getFileSystem(URI uri) throws IOException { Configuration config = new Configuration(); FileSystem fs = FileSystem.get(uri, config); if (fs instanceof LocalFileSystem) fs = ((LocalFileSystem)fs).getRaw(); return fs; }
public void initFileSystem(Configuration conf) throws IOException { Configuration confCopy = new Configuration(conf); fs = fsWorkingPath.getFileSystem(confCopy); // if it's local file system, use RawLocalFileSystem instead of // LocalFileSystem, the latter one doesn't support append. if (fs.getScheme().equals("file")) { fs = ((LocalFileSystem) fs).getRaw(); } }
@Deprecated public OnDiskMapOutput(TaskAttemptID mapId, TaskAttemptID reduceId, MergeManagerImpl<K,V> merger, long size, JobConf conf, MapOutputFile mapOutputFile, int fetcher, boolean primaryMapOutput) throws IOException { this(mapId, merger, size, conf, fetcher, primaryMapOutput, FileSystem.getLocal(conf).getRaw(), mapOutputFile.getInputFileForWrite(mapId.getTaskID(), size)); }
/** * Obtain the owner of the log dir. This is * determined by checking the job's log directory. */ static String obtainLogDirOwner(TaskAttemptID taskid) throws IOException { Configuration conf = new Configuration(); FileSystem raw = FileSystem.getLocal(conf).getRaw(); Path jobLogDir = new Path(getJobDir(taskid.getJobID()).getAbsolutePath()); FileStatus jobStat = raw.getFileStatus(jobLogDir); return jobStat.getOwner(); }
public OnDiskMapOutput(TaskAttemptID mapId, TaskAttemptID reduceId, MergeManagerImpl<K,V> merger, long size, JobConf conf, MapOutputFile mapOutputFile, int fetcher, boolean primaryMapOutput) throws IOException { this(mapId, reduceId, merger, size, conf, mapOutputFile, fetcher, primaryMapOutput, FileSystem.getLocal(conf).getRaw(), mapOutputFile.getInputFileForWrite(mapId.getTaskID(), size)); }