/** * Create an entry stream over the given queue at the given start position * @param logQueue the queue of WAL paths * @param fs {@link FileSystem} to use to create {@link Reader} for this stream * @param conf {@link Configuration} to use to create {@link Reader} for this stream * @param startPosition the position in the first WAL to start reading at * @param serverName the server name which all WALs belong to * @param metrics replication metrics * @throws IOException */ public WALEntryStream(PriorityBlockingQueue<Path> logQueue, Configuration conf, long startPosition, WALFileLengthProvider walFileLengthProvider, ServerName serverName, MetricsSource metrics) throws IOException { this.logQueue = logQueue; this.fs = CommonFSUtils.getWALFileSystem(conf); this.conf = conf; this.currentPositionOfEntry = startPosition; this.walFileLengthProvider = walFileLengthProvider; this.serverName = serverName; this.metrics = metrics; }
/** * @return List of all RegionServer WAL dirs; i.e. this.rootDir/HConstants.HREGION_LOGDIR_NAME. */ public FileStatus[] getWALDirPaths(final PathFilter filter) throws IOException { Path walDirPath = new Path(CommonFSUtils.getWALRootDir(conf), HConstants.HREGION_LOGDIR_NAME); FileStatus[] walDirForServerNames = FSUtils.listStatus( CommonFSUtils.getWALFileSystem(conf), walDirPath, filter); return walDirForServerNames == null? new FileStatus[0]: walDirForServerNames; }
private void recoverLease(final Configuration conf, final Path path) { try { final FileSystem dfs = CommonFSUtils.getWALFileSystem(conf); FSUtils fsUtils = FSUtils.getInstance(dfs, conf); fsUtils.recoverFileLease(dfs, path, conf, new CancelableProgressable() { @Override public boolean progress() { LOG.debug("recover WAL lease: " + path); return true; } }); } catch (IOException e) { LOG.warn("unable to recover lease for WAL: " + path, e); } }
@VisibleForTesting public WALProcedureStore(final Configuration conf, final Path walDir, final Path walArchiveDir, final LeaseRecovery leaseRecovery) throws IOException { this.conf = conf; this.leaseRecovery = leaseRecovery; this.walDir = walDir; this.walArchiveDir = walArchiveDir; this.fs = CommonFSUtils.getWALFileSystem(conf); this.enforceStreamCapability = conf.getBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, true); // Create the log directory for the procedure store if (!fs.exists(walDir)) { if (!fs.mkdirs(walDir)) { throw new IOException("Unable to mkdir " + walDir); } } // Now that it exists, set the log policy String storagePolicy = conf.get(HConstants.WAL_STORAGE_POLICY, HConstants.DEFAULT_WAL_STORAGE_POLICY); CommonFSUtils.setStoragePolicy(fs, walDir, storagePolicy); // Create archive dir up front. Rename won't work w/o it up on HDFS. if (this.walArchiveDir != null && !this.fs.exists(this.walArchiveDir)) { if (this.fs.mkdirs(this.walArchiveDir)) { LOG.debug("Created Procedure Store WAL archive dir {}", this.walArchiveDir); } else { LOG.warn("Failed create of {}", this.walArchiveDir); } } }
/** * Removes the empty Meta recovery WAL directory. * @param walFactoryId A unique identifier for WAL factory which was used by Filesystem to make a * Meta recovery WAL directory inside WAL directory path. */ private void removeHBCKMetaRecoveryWALDir(String walFactoryId) throws IOException { Path walLogDir = new Path(new Path(CommonFSUtils.getWALRootDir(getConf()), HConstants.HREGION_LOGDIR_NAME), walFactoryId); FileSystem fs = CommonFSUtils.getWALFileSystem(getConf()); FileStatus[] walFiles = FSUtils.listStatus(fs, walLogDir, null); if (walFiles == null || walFiles.length == 0) { LOG.info("HBCK meta recovery WAL directory is empty, removing it now."); if (!FSUtils.deleteDirectory(fs, walLogDir)) { LOG.warn("Couldn't clear the HBCK Meta recovery WAL directory " + walLogDir); } } }
@Override protected AsyncFSWAL createWAL() throws IOException { return new AsyncFSWAL(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf), getWALDirectoryName(factory.factoryId), getWALArchiveDirectoryName(conf, factory.factoryId), conf, listeners, true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null, eventLoopGroup, channelClass); }
@Override protected FSHLog createWAL() throws IOException { return new FSHLog(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf), getWALDirectoryName(factory.factoryId), getWALArchiveDirectoryName(conf, factory.factoryId), conf, listeners, true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null); }
FileSystem walFs = CommonFSUtils.getWALFileSystem(c); logFiles = BackupUtils.getFiles(walFs, logDir, logFiles, filter); logFiles = BackupUtils.getFiles(walFs, oldLogDir, logFiles, filter);
CommonFSUtils.getWALFileSystem(conf), ReplicationUtils.getRemoteWALFileSystem(conf, remoteWALDir), CommonFSUtils.getWALRootDir(conf),
private FSHLog createWAL() throws IOException { String logPrefix = factory.factoryId + WAL_FILE_NAME_DELIMITER + providerId; return new IOTestWAL(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf), AbstractFSWALProvider.getWALDirectoryName(factory.factoryId), HConstants.HREGION_OLDLOGDIR_NAME, conf, listeners, true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null); }
private FSHLog createWAL() throws IOException { String logPrefix = factory.factoryId + WAL_FILE_NAME_DELIMITER + providerId; return new IOTestWAL(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf), AbstractFSWALProvider.getWALDirectoryName(factory.factoryId), HConstants.HREGION_OLDLOGDIR_NAME, conf, listeners, true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null); }