@Override public OutputStream create(String path) throws IOException { return create(path, CreateOptions.defaults(mUfsConf).setCreateParent(true)); }
/** * Completes the current log and rotates in a new log. */ private void rotateLog() throws IOException { mDataOutputStream.close(); mJournalWriter.completeCurrentLog(); mRawOutputStream = mUfs.create(mCurrentLog.toString(), CreateOptions.defaults(ServerConfiguration.global()).setEnsureAtomic(false) .setCreateParent(true)); LOG.info("Opened current log file: {}", mCurrentLog); mDataOutputStream = new DataOutputStream(mRawOutputStream); } }
/** * @param ufs the under storage holding the journal * @param log the location to write the log to * @param journalFormatter the journal formatter to use when writing journal entries * @param journalWriter the journal writer to use to get journal entry sequence numbers and * complete the log when it needs to be rotated */ public EntryOutputStream(UnderFileSystem ufs, URI log, JournalFormatter journalFormatter, UfsJournalWriter journalWriter) throws IOException { mUfs = ufs; mCurrentLog = log; mJournalFormatter = journalFormatter; mJournalWriter = journalWriter; mMaxLogSize = ServerConfiguration.getBytes(PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX); mRawOutputStream = mUfs.create(mCurrentLog.toString(), CreateOptions.defaults(ServerConfiguration.global()) .setEnsureAtomic(false).setCreateParent(true)); LOG.info("Opened current log file: {}", mCurrentLog); mDataOutputStream = new DataOutputStream(mRawOutputStream); }
private void createNewLogFile() throws IOException { URI newLog = UfsJournalFile .encodeLogFileLocation(mJournal, mNextSequenceNumber, UfsJournal.UNKNOWN_SEQUENCE_NUMBER); UfsJournalFile currentLog = UfsJournalFile.createLogFile(newLog, mNextSequenceNumber, UfsJournal.UNKNOWN_SEQUENCE_NUMBER); OutputStream outputStream = mUfs.create(currentLog.getLocation().toString(), CreateOptions.defaults(ServerConfiguration.global()).setEnsureAtomic(false) .setCreateParent(true)); mJournalOutputStream = new JournalOutputStream(currentLog, outputStream); LOG.info("Created current log file: {}", currentLog); }
@Test public void createParent() throws IOException { String testFile = PathUtils.concatPath(mUnderfsAddress, "createParent/testFile"); OutputStream o = mUfs.create(testFile, CreateOptions.defaults(mConfiguration) .setCreateParent(true)); o.close(); assertTrue(mUfs.exists(testFile)); }
@Test public void createNoParent() throws IOException { // Run the test only for local UFS. Other UFSs succeed if no parents are present Assume.assumeTrue(UnderFileSystemUtils.isLocal(mUfs)); mThrown.expect(IOException.class); String testFile = PathUtils.concatPath(mUnderfsAddress, "createNoParent/testFile"); OutputStream o = mUfs.create(testFile, CreateOptions.defaults(mConfiguration) .setCreateParent(false)); o.close(); }
/** * Tests getting and setting fields. */ @Test public void fields() { Random random = new Random(); boolean createParent = random.nextBoolean(); boolean ensureAtomic = random.nextBoolean(); String owner = CommonUtils.randomAlphaNumString(10); String group = CommonUtils.randomAlphaNumString(10); Mode mode = new Mode((short) random.nextInt()); CreateOptions options = CreateOptions.defaults(mConfiguration); options.setCreateParent(createParent); options.setEnsureAtomic(ensureAtomic); options.setOwner(owner); options.setGroup(group); options.setMode(mode); assertEquals(createParent, options.getCreateParent()); assertEquals(ensureAtomic, options.isEnsureAtomic()); assertEquals(owner, options.getOwner()); assertEquals(group, options.getGroup()); assertEquals(mode, options.getMode()); }
/** * Creates a UFS block and initialize it with bytes read from block store. * * @param context context of this request */ private void createUfsBlock(BlockWriteRequestContext context) throws Exception { BlockWriteRequest request = context.getRequest(); Protocol.CreateUfsBlockOptions createUfsBlockOptions = request.getCreateUfsBlockOptions(); UfsManager.UfsClient ufsClient = mUfsManager.get(createUfsBlockOptions.getMountId()); alluxio.resource.CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource(); context.setUfsResource(ufsResource); String ufsString = MetricsSystem.escape(ufsClient.getUfsMountPointUri()); String ufsPath = BlockUtils.getUfsBlockPath(ufsClient, request.getId()); UnderFileSystem ufs = ufsResource.get(); // Set the atomic flag to be true to ensure only the creation of this file is atomic on close. OutputStream ufsOutputStream = ufs.create(ufsPath, CreateOptions.defaults(ServerConfiguration.global()).setEnsureAtomic(true) .setCreateParent(true)); context.setOutputStream(ufsOutputStream); context.setUfsPath(ufsPath); String counterName = Metric.getMetricNameWithTags(WorkerMetrics.BYTES_WRITTEN_UFS, WorkerMetrics.TAG_UFS, ufsString); String meterName = Metric.getMetricNameWithTags(WorkerMetrics.BYTES_WRITTEN_UFS_THROUGHPUT, WorkerMetrics.TAG_UFS, ufsString); context.setCounter(MetricsSystem.counter(counterName)); context.setMeter(MetricsSystem.meter(meterName)); }
/** * Completes the current log and rotates in a new log. */ private void rotateLog() throws IOException { mDataOutputStream.close(); mJournalWriter.completeCurrentLog(); mRawOutputStream = mUfs.create(mCurrentLog.toString(), CreateOptions.defaults().setEnsureAtomic(false).setCreateParent(true)); LOG.info("Opened current log file: {}", mCurrentLog); mDataOutputStream = new DataOutputStream(mRawOutputStream); } }
@Override public OutputStream create(String path) throws IOException { return create(path, CreateOptions.defaults().setCreateParent(true)); }
/** * @param ufs the under storage holding the journal * @param log the location to write the log to * @param journalFormatter the journal formatter to use when writing journal entries * @param journalWriter the journal writer to use to get journal entry sequence numbers and * complete the log when it needs to be rotated */ public EntryOutputStream(UnderFileSystem ufs, URI log, JournalFormatter journalFormatter, UfsJournalWriter journalWriter) throws IOException { mUfs = ufs; mCurrentLog = log; mJournalFormatter = journalFormatter; mJournalWriter = journalWriter; mMaxLogSize = Configuration.getBytes(PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX); mRawOutputStream = mUfs.create(mCurrentLog.toString(), CreateOptions.defaults().setEnsureAtomic(false).setCreateParent(true)); LOG.info("Opened current log file: {}", mCurrentLog); mDataOutputStream = new DataOutputStream(mRawOutputStream); }
private void createNewLogFile() throws IOException { URI newLog = UfsJournalFile .encodeLogFileLocation(mJournal, mNextSequenceNumber, UfsJournal.UNKNOWN_SEQUENCE_NUMBER); UfsJournalFile currentLog = UfsJournalFile.createLogFile(newLog, mNextSequenceNumber, UfsJournal.UNKNOWN_SEQUENCE_NUMBER); OutputStream outputStream = mUfs.create(currentLog.getLocation().toString(), CreateOptions.defaults().setEnsureAtomic(false).setCreateParent(true)); mJournalOutputStream = new JournalOutputStream(currentLog, outputStream); LOG.info("Created current log file: {}", currentLog); }