@SuppressWarnings("deprecation") @Test public void testGetTokensForNamenodes() throws IOException, URISyntaxException { Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "test/build/data")); // ick, but need fq path minus file:/ String binaryTokenFile = FileSystem.getLocal(conf) .makeQualified(new Path(TEST_ROOT_DIR, "tokenFile")).toUri() .getPath(); MockFileSystem fs1 = createFileSystemForServiceName("service1"); Credentials creds = new Credentials(); Token<?> token1 = fs1.getDelegationToken(renewer); creds.addToken(token1.getService(), token1); // wait to set, else the obtain tokens call above will fail with FNF conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile); creds.writeTokenStorageFile(new Path(binaryTokenFile), conf); TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); String fs_addr = fs1.getCanonicalServiceName(); Token<?> nnt = TokenCache.getDelegationToken(creds, fs_addr); assertNotNull("Token for nn is null", nnt); } }
? FileSystem.getLocal(conf).makeQualified( new Path(TEST_ROOT_DIR, "tokenFile")).toString() : FileSystem.getLocal(conf).makeQualified( new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath();
/** * Returns the local temporary directory. * @param localFileSystem the local file system * @return the output path (must be on local fs), or {@code null} if not defined * @throws IOException if failed to compute the path * @throws IllegalArgumentException if some parameters were {@code null} */ public static Path getLocalTemporaryDirectory(LocalFileSystem localFileSystem) throws IOException { if (localFileSystem == null) { throw new IllegalArgumentException("localFileSystem must not be null"); //$NON-NLS-1$ } Configuration conf = localFileSystem.getConf(); if (conf == null) { return null; } String path = conf.get(KEY_LOCAL_TEMPDIR); if (path == null) { return null; } LocalFileSystem fs = FileSystem.getLocal(conf); Path result = fs.makeQualified(new Path(path)); return result; }
private static void mergeBinaryTokens(Credentials creds, Configuration conf) { String binaryTokenFilename = conf.get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY); if (binaryTokenFilename != null) { Credentials binary; try { binary = Credentials.readTokenStorageFile( FileSystem.getLocal(conf).makeQualified( new Path(binaryTokenFilename)), conf); } catch (IOException e) { throw new RuntimeException(e); } // supplement existing tokens with the tokens in the binary file creds.mergeAll(binary); } }
private static void mergeBinaryTokens(Credentials creds, Configuration conf) { String binaryTokenFilename = conf.get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY); if (binaryTokenFilename != null) { Credentials binary; try { binary = Credentials.readTokenStorageFile( FileSystem.getLocal(conf).makeQualified( new Path(binaryTokenFilename)), conf); } catch (IOException e) { throw new RuntimeException(e); } // supplement existing tokens with the tokens in the binary file creds.mergeAll(binary); } }
private static void mergeBinaryTokens(Credentials creds, Configuration conf) { String binaryTokenFilename = conf.get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY); if (binaryTokenFilename != null) { Credentials binary; try { binary = Credentials.readTokenStorageFile( FileSystem.getLocal(conf).makeQualified( new Path(binaryTokenFilename)), conf); } catch (IOException e) { throw new RuntimeException(e); } // supplement existing tokens with the tokens in the binary file creds.mergeAll(binary); } }
private static void mergeBinaryTokens(Credentials creds, Configuration conf) { String binaryTokenFilename = conf.get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY); if (binaryTokenFilename != null) { Credentials binary; try { binary = Credentials.readTokenStorageFile( FileSystem.getLocal(conf).makeQualified( new Path(binaryTokenFilename)), conf); } catch (IOException e) { throw new RuntimeException(e); } // supplement existing tokens with the tokens in the binary file creds.mergeAll(binary); } }
/** * Retrieves paths to cached files. * @param conf - MapReduce Configuration * @return Path[] of Cached Files * @throws IOException - IO Exception * @throws IllegalStateException if no cache files are found */ public static Path[] getCachedFiles(Configuration conf) throws IOException { LocalFileSystem localFs = FileSystem.getLocal(conf); Path[] cacheFiles = DistributedCache.getLocalCacheFiles(conf); URI[] fallbackFiles = DistributedCache.getCacheFiles(conf); // fallback for local execution if (cacheFiles == null) { Preconditions.checkState(fallbackFiles != null, "Unable to find cached files!"); cacheFiles = new Path[fallbackFiles.length]; for (int n = 0; n < fallbackFiles.length; n++) { cacheFiles[n] = new Path(fallbackFiles[n].getPath()); } } else { for (int n = 0; n < cacheFiles.length; n++) { cacheFiles[n] = localFs.makeQualified(cacheFiles[n]); // fallback for local execution if (!localFs.exists(cacheFiles[n])) { cacheFiles[n] = new Path(fallbackFiles[n].getPath()); } } } Preconditions.checkState(cacheFiles.length > 0, "Unable to find cached files!"); return cacheFiles; }
/** * Retrieves paths to cached files. * @param conf - MapReduce Configuration * @return Path[] of Cached Files * @throws IOException - IO Exception * @throws IllegalStateException if no cache files are found */ public static Path[] getCachedFiles(Configuration conf) throws IOException { LocalFileSystem localFs = FileSystem.getLocal(conf); Path[] cacheFiles = DistributedCache.getLocalCacheFiles(conf); URI[] fallbackFiles = DistributedCache.getCacheFiles(conf); // fallback for local execution if (cacheFiles == null) { Preconditions.checkState(fallbackFiles != null, "Unable to find cached files!"); cacheFiles = new Path[fallbackFiles.length]; for (int n = 0; n < fallbackFiles.length; n++) { cacheFiles[n] = new Path(fallbackFiles[n].getPath()); } } else { for (int n = 0; n < cacheFiles.length; n++) { cacheFiles[n] = localFs.makeQualified(cacheFiles[n]); // fallback for local execution if (!localFs.exists(cacheFiles[n])) { cacheFiles[n] = new Path(fallbackFiles[n].getPath()); } } } Preconditions.checkState(cacheFiles.length > 0, "Unable to find cached files!"); return cacheFiles; }
/** * Retrieves paths to cached files. * @param conf - MapReduce Configuration * @return Path[] of Cached Files * @throws IOException - IO Exception * @throws IllegalStateException if no cache files are found */ public static Path[] getCachedFiles(Configuration conf) throws IOException { LocalFileSystem localFs = FileSystem.getLocal(conf); Path[] cacheFiles = DistributedCache.getLocalCacheFiles(conf); URI[] fallbackFiles = DistributedCache.getCacheFiles(conf); // fallback for local execution if (cacheFiles == null) { Preconditions.checkState(fallbackFiles != null, "Unable to find cached files!"); cacheFiles = new Path[fallbackFiles.length]; for (int n = 0; n < fallbackFiles.length; n++) { cacheFiles[n] = new Path(fallbackFiles[n].getPath()); } } else { for (int n = 0; n < cacheFiles.length; n++) { cacheFiles[n] = localFs.makeQualified(cacheFiles[n]); // fallback for local execution if (!localFs.exists(cacheFiles[n])) { cacheFiles[n] = new Path(fallbackFiles[n].getPath()); } } } Preconditions.checkState(cacheFiles.length > 0, "Unable to find cached files!"); return cacheFiles; }
private Path createTmpFile() throws IOException { File tmp = File.createTempFile(getClass().getSimpleName(), ".tmp"); tmp.deleteOnExit(); tmp.delete(); // it prevents accessing HDFS namenode of TajoTestingCluster. LocalFileSystem localFS = LocalFileSystem.getLocal(new Configuration()); return localFS.makeQualified(new Path(tmp.getPath())); }
@BeforeClass public static void setup() throws Exception { conf = new Configuration(); shell = new FsShell(conf); lfs = FileSystem.getLocal(conf); testRootDir = lfs.makeQualified(new Path( System.getProperty("test.build.data","test/build/data"), "testShellCopy")); lfs.mkdirs(testRootDir); srcPath = new Path(testRootDir, "srcFile"); dstPath = new Path(testRootDir, "dstFile"); }
@BeforeClass public static void setup() throws Exception { conf = new Configuration(); shell = new FsShell(conf); lfs = FileSystem.getLocal(conf); testRootDir = lfs.makeQualified(new Path( System.getProperty("test.build.data","test/build/data"), "testShellCopy")); lfs.mkdirs(testRootDir); srcPath = new Path(testRootDir, "srcFile"); dstPath = new Path(testRootDir, "dstFile"); }
/** * Creates a sequence file. * @throws Exception if failed */ @Test public void write() throws Exception { Path path = new Path("testing"); Text key = new Text(); Text value = new Text(); try (OutputStream out = new FileOutputStream(fs.pathToFile(path)); SequenceFile.Writer writer = SequenceFileUtil.openWriter( new BufferedOutputStream(out), conf, key.getClass(), value.getClass(), null)) { key.set("Hello"); value.set("World"); writer.append(key, value); } key.clear(); value.clear(); try (SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(fs.makeQualified(path)))) { assertThat(reader.next(key, value), is(true)); assertThat(key.toString(), is("Hello")); assertThat(value.toString(), is("World")); assertThat(reader.next(key, value), is(false)); } }
/** * invalid compressed output. * @throws Exception if failed */ @Test public void output_compressed_invalid() throws Exception { LocalFileSystem fs = FileSystem.getLocal(conf); Path path = new Path(folder.newFile("testing").toURI()); format.getConf().set(SequenceFileFormat.KEY_COMPRESSION_CODEC, "__INVALID__"); try (ModelOutput<StringOption> out = format.createOutput(StringOption.class, fs, path, new Counter())) { out.write(new StringOption("Hello, world!")); } try (SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(fs.makeQualified(path)))) { assertThat(reader.getCompressionCodec(), is(nullValue())); } }
/** * compressed output. * @throws Exception if failed */ @Test public void output_compressed_conf() throws Exception { LocalFileSystem fs = FileSystem.getLocal(conf); Path path = new Path(folder.newFile("testing").toURI()); format.getConf().set(SequenceFileFormat.KEY_COMPRESSION_CODEC, DefaultCodec.class.getName()); try (ModelOutput<StringOption> out = format.createOutput(StringOption.class, fs, path, new Counter())) { out.write(new StringOption("Hello, world!")); } try (SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(fs.makeQualified(path)))) { assertThat(reader.getCompressionCodec(), instanceOf(DefaultCodec.class)); } }
/** * Reads a sequence file. * @throws Exception if failed */ @Test public void read_new() throws Exception { Path path = new Path("testing"); Text key = new Text(); Text value = new Text(); try (SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(fs.makeQualified(path)), SequenceFile.Writer.keyClass(key.getClass()), SequenceFile.Writer.valueClass(value.getClass()))) { key.set("Hello"); value.set("World"); writer.append(key, value); } key.clear(); value.clear(); FileStatus status = fs.getFileStatus(path); try (InputStream in = new FileInputStream(fs.pathToFile(path)); SequenceFile.Reader reader = SequenceFileUtil.openReader(in, status.getLen(), conf)) { assertThat(reader.next(key, value), is(true)); assertThat(key.toString(), is("Hello")); assertThat(value.toString(), is("World")); assertThat(reader.next(key, value), is(false)); } }
/** * compressed output. * @throws Exception if failed */ @Test public void output_no_compressed() throws Exception { LocalFileSystem fs = FileSystem.getLocal(conf); Path path = new Path(folder.newFile("testing.gz").toURI()); try (ModelOutput<StringOption> out = format.codec(null) .createOutput(StringOption.class, fs, path, new Counter())) { out.write(new StringOption("Hello, world!")); } try (SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(fs.makeQualified(path)))) { assertThat(reader.getCompressionCodec(), is(nullValue())); } }
/** * Reads a large sequence file. * @throws Exception if failed */ @Test public void read_large() throws Exception { Path path = new Path("large"); LongWritable key = new LongWritable(); LongWritable value = new LongWritable(); try (SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(fs.makeQualified(path)), SequenceFile.Writer.keyClass(key.getClass()), SequenceFile.Writer.valueClass(value.getClass()))) { for (long i = 0; i < 300000; i++) { key.set(i); value.set(i + 1); writer.append(key, value); } } FileStatus status = fs.getFileStatus(path); try (InputStream in = new FileInputStream(fs.pathToFile(path)); SequenceFile.Reader reader = SequenceFileUtil.openReader(in, status, conf)) { for (long i = 0; i < 300000; i++) { assertThat(reader.next(key, value), is(true)); assertThat(key.get(), is(i)); assertThat(value.get(), is(i + 1)); } assertThat(reader.next(key, value), is(false)); } }
/** * compressed output. * @throws Exception if failed */ @Test public void output_compressed() throws Exception { LocalFileSystem fs = FileSystem.getLocal(conf); Path path = new Path(folder.newFile("testing").toURI()); try (ModelOutput<StringOption> out = format.codec(new DefaultCodec()) .createOutput(StringOption.class, fs, path, new Counter())) { out.write(new StringOption("Hello, world!")); } try (SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(fs.makeQualified(path)))) { assertThat(reader.getCompressionCodec(), instanceOf(DefaultCodec.class)); } }