private void writeFile(FileSystem fs) throws IOException, AlluxioException { for (int i = 0; i < mNumFiles; i++) { ByteBuffer buf = ByteBuffer.allocate(80); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumFiles; k++) { buf.putInt(k); } buf.flip(); AlluxioURI filePath = new AlluxioURI(mFileFolder + "/part-" + i); LOG.debug("Writing data to {}", filePath); OutputStream os = fs.createFile(filePath, CreateFilePOptions.newBuilder().setRecursive(true).build()); os.write(buf.array()); os.close(); } }
.setFileWriteLocationPolicy(mFsContext.getConf().get( PropertyKey.USER_FILE_COPY_FROM_LOCAL_WRITE_LOCATION_POLICY)) .build(); os = closer.register(mFileSystem.createFile(dstPath, createOptions)); FileInputStream in = closer.register(new FileInputStream(src));
/** * Writes a partition. */ public void writePartition() throws IOException, AlluxioException { if (sDebugMode) { mBuf.flip(); LOG.info(FormatUtils.byteBufferToString(mBuf)); } mBuf.flip(); for (int pId = mLeft; pId < mRight; pId++) { final long startTimeMs = System.currentTimeMillis(); FileOutStream os = mFileSystem.createFile(new AlluxioURI(sFileName + (pId + sBaseFileNumber)), CreateFilePOptions.newBuilder().setRecursive(true).build()); for (int k = 0; k < sBlocksPerFile; k++) { mBuf.putInt(0, k + mWorkerId); os.write(mBuf.array()); } os.close(); logPerIteration(startTimeMs, pId, "th WriteAlluxioFile @ Worker ", pId); } }
.setWriteType(WritePType.CACHE_THROUGH).setRecursive(true).build(); FileOutStream os = fileSystem.createFile(outputPath, options); System.out.println("done");
CreateFilePOptions.newBuilder().setWriteType(WritePType.NONE).build();
.setMode(new Mode(permission.toShort()).toProto()).setRecursive(true).build();
/** * @param command the move command to execute * @param writeType the write type to use for the moved file * @param fileSystem the Alluxio file system */ private static void move(MoveCommand command, WritePType writeType, FileSystem fileSystem) throws Exception { String source = command.getSource(); String destination = command.getDestination(); LOG.debug("Moving {} to {}", source, destination); CreateFilePOptions createOptions = CreateFilePOptions.newBuilder().setWriteType(writeType).build(); try (FileOutStream out = fileSystem.createFile(new AlluxioURI(destination), createOptions)) { try (FileInStream in = fileSystem.openFile(new AlluxioURI(source))) { IOUtils.copy(in, out); } catch (Throwable t) { try { out.cancel(); } catch (Throwable t2) { t.addSuppressed(t2); } throw t; } } fileSystem.delete(new AlluxioURI(source)); }
@Override public void apply(FileSystem fs) throws Exception { try (FileOutStream out = fs.createFile(FILE, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setWriteType(WritePType.ASYNC_THROUGH).build())) { out.write("test".getBytes()); } // Nested file try (FileOutStream out = fs.createFile(NESTED_FILE, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setWriteType(WritePType.ASYNC_THROUGH).setRecursive(true).build())) { out.write("test".getBytes()); } CommonUtils.waitFor("files to be persisted", () -> { try { return fs.getStatus(FILE).isPersisted() && fs.getStatus(NESTED_FILE).isPersisted(); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }); }
@Override public FSDataOutputStream append(Path path, int bufferSize, Progressable progress) throws IOException { LOG.debug("append({}, {}, {})", path, bufferSize, progress); if (mStatistics != null) { mStatistics.incrementWriteOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); try { if (mFileSystem.exists(uri)) { throw new IOException(ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(uri)); } return new FSDataOutputStream( mFileSystem.createFile(uri, CreateFilePOptions.newBuilder().setRecursive(true).build()), mStatistics); } catch (AlluxioException e) { throw new IOException(e); } }
/** * @return Master side defaults for {@link CreateFilePOptions} */ public static CreateFilePOptions createFileDefaults() { return CreateFilePOptions.newBuilder() .setCommonOptions(commonDefaults()) .setRecursive(false) .setBlockSizeBytes(ServerConfiguration.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT)) .setFileWriteLocationPolicy( ServerConfiguration.get(PropertyKey.USER_FILE_WRITE_LOCATION_POLICY)) .setWriteTier(ServerConfiguration.getInt(PropertyKey.USER_FILE_WRITE_TIER_DEFAULT)) .setWriteType(ServerConfiguration .getEnum(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.class).toProto()) .setMode(ModeUtils.applyFileUMask(Mode.defaults(), ServerConfiguration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK)).toProto()) .setReplicationDurable(ServerConfiguration .getInt(PropertyKey.USER_FILE_REPLICATION_DURABLE)) .setReplicationMin(ServerConfiguration.getInt(PropertyKey.USER_FILE_REPLICATION_MIN)) .setReplicationMax(ServerConfiguration.getInt(PropertyKey.USER_FILE_REPLICATION_MAX)) .build(); }
/** * <code>optional .alluxio.grpc.file.CreateFilePOptions options = 2;</code> */ public Builder setOptions( alluxio.grpc.CreateFilePOptions.Builder builderForValue) { if (optionsBuilder_ == null) { options_ = builderForValue.build(); onChanged(); } else { optionsBuilder_.setMessage(builderForValue.build()); } bitField0_ |= 0x00000002; return this; } /**
/** * Creates a simple file with {@code len} bytes. * * @param fs a {@link FileSystem} handler * @param fileName the name of the file to be created * @param writeType {@link WritePType} used to create the file * @param len file size * @param blockCapacityByte block size of the file */ public static void createByteFile(FileSystem fs, String fileName, WritePType writeType, int len, long blockCapacityByte) { CreateFilePOptions options = CreateFilePOptions.newBuilder().setWriteType(writeType) .setBlockSizeBytes(blockCapacityByte).setRecursive(true).build(); createByteFile(fs, new AlluxioURI(fileName), options, len); }
private FileOutStream createFile(FileSystem fileSystem, AlluxioURI filePath, boolean deleteIfExists) throws IOException, AlluxioException { CreateFilePOptions options = CreateFilePOptions.newBuilder().setWriteType(mWriteType.toProto()) .setRecursive(true).build(); if (!fileSystem.exists(filePath)) { // file doesn't exist yet, so create it return fileSystem.createFile(filePath, options); } else if (deleteIfExists) { // file exists, so delete it and recreate fileSystem.delete(filePath); return fileSystem.createFile(filePath, options); } // file exists and deleteIfExists is false throw new FileAlreadyExistsException("File exists and deleteIfExists is false"); }
/** * @param filePath the path for the files * @param readType the {@link ReadPType} * @param writeType the {@link WritePType} * @param fsContext the {@link FileSystemContext } to use for client operations */ public BasicOperations(AlluxioURI filePath, ReadType readType, WriteType writeType, FileSystemContext fsContext) { mFilePath = filePath; mReadOptions = OpenFilePOptions.newBuilder().setReadType(readType.toProto()).build(); mWriteOptions = CreateFilePOptions.newBuilder().setWriteType(writeType.toProto()) .setRecursive(true).build(); mFsContext = fsContext; }
/** * Creates a simple file with {@code len} bytes. * * @param fs a {@link FileSystem} handler * @param fileURI URI of the file * @param writeType {@link WritePType} used to create the file * @param len file size */ public static void createByteFile(FileSystem fs, AlluxioURI fileURI, WritePType writeType, int len) { CreateFilePOptions options = CreateFilePOptions.newBuilder().setRecursive(true).setWriteType(writeType).build(); createByteFile(fs, fileURI, options, len); }
/** * Tests that the worker writes with the specified write type. */ @Test @Ignore // TODO(ggezer) Fix. public void writeTypeTest() throws Exception { runTask(TEST_SOURCE, TEST_SOURCE, TEST_DESTINATION, WriteType.CACHE_THROUGH); verify(mMockFileSystem).createFile(eq(new AlluxioURI(TEST_DESTINATION)), Matchers .eq(CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH)).build()); runTask(TEST_SOURCE, TEST_SOURCE, TEST_DESTINATION, WriteType.MUST_CACHE); verify(mMockFileSystem).createFile(eq(new AlluxioURI(TEST_DESTINATION)), Matchers .eq(CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE)).build()); }
@Override public void apply(FileSystem fs) throws Exception { Utils.createFile(fs, PATH); Utils.createFile(fs, NESTED); Utils.createFile(fs, MODE, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setRecursive(true).setMode(TEST_MODE.toProto()).build()); Utils.createFile(fs, THROUGH, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setRecursive(true).setWriteType(WritePType.THROUGH).build()); Utils.createFile(fs, TTL, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true) .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(TEST_TTL) .setTtlAction(alluxio.grpc.TtlAction.FREE)) .build()); }
/** * Merges and embeds the given {@link CreateFilePOptions} with the corresponding master options. * * @param optionsBuilder Builder for proto {@link CreateFilePOptions} to embed * @return the instance of {@link CreateFileContext} with default values for master */ public static CreateFileContext defaults(CreateFilePOptions.Builder optionsBuilder) { CreateFilePOptions masterOptions = FileSystemMasterOptions.createFileDefaults(); CreateFilePOptions.Builder mergedOptionsBuilder = masterOptions.toBuilder().mergeFrom(optionsBuilder.build()); return new CreateFileContext(mergedOptionsBuilder); }
/** * Creates a file at the given path. * * @param fs a filesystem client * @param path the file path */ public static void createFile(FileSystem fs, AlluxioURI path) throws Exception { try (FileOutStream out = fs.createFile(path, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setRecursive(true).build())) { out.write("test".getBytes()); } }