/** * Construct the preferred type of SequenceFile Writer. * @param fc The context for the specified file. * @param conf The configuration. * @param name The name of the file. * @param keyClass The 'key' type. * @param valClass The 'value' type. * @param compressionType The compression type. * @param codec The compression codec. * @param metadata The metadata of the file. * @param createFlag gives the semantics of create: overwrite, append etc. * @param opts file creation options; see {@link CreateOpts}. * @return Returns the handle to the constructed SequenceFile Writer. * @throws IOException */ public static Writer createWriter(FileContext fc, Configuration conf, Path name, Class keyClass, Class valClass, CompressionType compressionType, CompressionCodec codec, Metadata metadata, final EnumSet<CreateFlag> createFlag, CreateOpts... opts) throws IOException { return createWriter(conf, fc.create(name, createFlag, opts), keyClass, valClass, compressionType, codec, metadata).ownStream(); }
@Override public FSDataOutputStream build() throws IOException { final EnumSet<CreateFlag> flags = getFlags(); List<CreateOpts> createOpts = new ArrayList<>(Arrays.asList( CreateOpts.blockSize(getBlockSize()), CreateOpts.bufferSize(getBufferSize()), CreateOpts.repFac(getReplication()), CreateOpts.perms(getPermission()) )); if (getChecksumOpt() != null) { createOpts.add(CreateOpts.checksumParam(getChecksumOpt())); } if (getProgress() != null) { createOpts.add(CreateOpts.progress(getProgress())); } if (isRecursive()) { createOpts.add(CreateOpts.createParent()); } return fc.create(getPath(), flags, createOpts.toArray(new CreateOpts[0])); } }
EnumSet.of(CreateFlag.CREATE); InputStream in = open(qSrc); try (OutputStream out = create(qDst, createFlag)) { IOUtils.copyBytes(in, out, conf, true); } finally {
@Override public FSDataOutputStream create(Path f, EnumSet<CreateFlag> createFlag, CreateOpts... opts) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, IOException { return fc.create(f, createFlag, opts); }
@Override public FSDataOutputStream run() throws Exception { fc = FileContext.getFileContext(remoteAppLogFile.toUri(), conf); fc.setUMask(APP_LOG_FILE_UMASK); return fc.create( remoteAppLogFile, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), new Options.CreateOpts[] {}); } });
public void writeLocalWrapperScript(Path launchDst, Path pidFile) throws IOException { DataOutputStream out = null; PrintStream pout = null; try { out = lfs.create(wrapperScriptPath, EnumSet.of(CREATE, OVERWRITE)); pout = new PrintStream(out, false, "UTF-8"); writeLocalWrapperScript(launchDst, pidFile, pout); } finally { IOUtils.cleanup(LOG, pout, out); } }
@Override public FSDataOutputStream run() throws Exception { fc = FileContext.getFileContext(conf); fc.setUMask(APP_LOG_FILE_UMASK); return fc.create( remoteAppLogFile, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), new Options.CreateOpts[] {}); } });
public static void createFile(FileContext fc, Path path, int numBlocks, CreateOpts... options) throws IOException { BlockSize blockSizeOpt = (BlockSize) CreateOpts.getOpt(CreateOpts.BlockSize.class, options); long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue() : DEFAULT_BLOCK_SIZE; FSDataOutputStream out = fc.create(path, EnumSet.of(CreateFlag.CREATE), options); byte[] data = getFileData(numBlocks, blockSize); out.write(data, 0, data.length); out.close(); }
static void createFile(FileContext files, Path p, int len, Random r) throws IOException { FSDataOutputStream out = null; try { byte[] bytes = new byte[len]; out = files.create(p, EnumSet.of(CREATE, OVERWRITE)); r.nextBytes(bytes); out.write(bytes); } finally { if (out != null) out.close(); } }
static void createFile(FileContext files, Path p, int len, Random r) throws IOException { FSDataOutputStream out = null; try { byte[] bytes = new byte[len]; out = files.create(p, EnumSet.of(CREATE, OVERWRITE)); r.nextBytes(bytes); out.write(bytes); } finally { if (out != null) out.close(); } }
public FSJsonLineFile(FileContext fileContext, Path path, FsPermission permission) throws IOException { this.os = fileContext.create(path, EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND), Options.CreateOpts.perms(permission)); this.objectMapper = (new JSONSerializationProvider()).getContext(null); }
public static void writeFile(FileContext fc, Path path, byte b[]) throws IOException { FSDataOutputStream out = fc.create(path,EnumSet.of(CreateFlag.CREATE), CreateOpts.createParent()); out.write(b); out.close(); }
public static long createFile(FileContext fc, Path path, int numBlocks, CreateOpts... options) throws IOException { BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options); long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue() : DEFAULT_BLOCK_SIZE; FSDataOutputStream out = fc.create(path, EnumSet.of(CreateFlag.CREATE), options); byte[] data = getFileData(numBlocks, blockSize); out.write(data, 0, data.length); out.close(); return data.length; }
public static void appendToFile(FileContext fc, Path path, int numBlocks, CreateOpts... options) throws IOException { BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options); long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue() : DEFAULT_BLOCK_SIZE; FSDataOutputStream out; out = fc.create(path, EnumSet.of(CreateFlag.APPEND)); byte[] data = getFileData(numBlocks, blockSize); out.write(data, 0, data.length); out.close(); }
@Test public void testOutputStreamClosedTwice() throws IOException { //HADOOP-4760 according to Closeable#close() closing already-closed //streams should have no effect. Path src = getTestRootPath(fc, "test/hadoop/file"); FSDataOutputStream out = fc.create(src, EnumSet.of(CREATE), Options.CreateOpts.createParent()); out.writeChar('H'); //write some data out.close(); out.close(); }
@Test(expected=HadoopIllegalArgumentException.class) public void testEmptyCreateFlag() throws IOException { Path p = getTestRootPath(fc, "test/file"); fc.create(p, EnumSet.noneOf(CreateFlag.class)); Assert.fail("Excepted exception not thrown"); }
@Test(expected=HadoopIllegalArgumentException.class) public void testCreateFlagAppendCreateOverwrite() throws IOException { Path p = getTestRootPath(fc, "test/nonExistent"); fc.create(p, EnumSet.of(CREATE, APPEND, OVERWRITE)); Assert.fail("Excepted exception not thrown"); }
@Test(expected=HadoopIllegalArgumentException.class) public void testCreateFlagAppendOverwrite() throws IOException { Path p = getTestRootPath(fc, "test/nonExistent"); fc.create(p, EnumSet.of(APPEND, OVERWRITE)); Assert.fail("Excepted exception not thrown"); }
@Test(expected=FileAlreadyExistsException.class) public void testCreateFlagCreateExistingFile() throws IOException { Path p = getTestRootPath(fc, "test/testCreateFlagCreateExistingFile"); createFile(p); fc.create(p, EnumSet.of(CREATE)); Assert.fail("Excepted exception not thrown"); }
@Test public void testCreateFlagCreateAppendExistingFile() throws IOException { Path p = getTestRootPath(fc, "test/testCreateFlagCreateAppendExistingFile"); createFile(p); FSDataOutputStream out = fc.create(p, EnumSet.of(CREATE, APPEND)); writeData(fc, p, out, data, 2*data.length); }