/** * Constructor for PinListSync * * @param blockDataManager the blockDataManager this syncer is updating to * @param masterClient the Tachyon master client */ public PinListSync(BlockDataManager blockDataManager, WorkerFileSystemMasterClient masterClient) { mBlockDataManager = blockDataManager; TachyonConf conf = WorkerContext.getConf(); mMasterClient = masterClient; mSyncIntervalMs = conf.getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS); mSyncTimeoutMs = conf.getInt(Constants.WORKER_BLOCK_HEARTBEAT_TIMEOUT_MS); mRunning = true; }
/** * @param master the master to apply the journal entries to * @param journal the journal to tail */ public JournalTailerThread(Master master, Journal journal) { mMaster = Preconditions.checkNotNull(master); mJournal = Preconditions.checkNotNull(journal); TachyonConf conf = MasterContext.getConf(); mShutdownQuietWaitTimeMs = conf.getInt( Constants.MASTER_JOURNAL_TAILER_SHUTDOWN_QUIET_WAIT_TIME_MS); mJournalTailerSleepTimeMs = conf.getInt(Constants.MASTER_JOURNAL_TAILER_SLEEP_TIME_MS); }
/** * Constructor for BlockMasterSync * * @param blockDataManager the blockDataManager this syncer is updating to * @param workerAddress the net address of the worker * @param masterClient the Tachyon master client */ BlockMasterSync(BlockDataManager blockDataManager, NetAddress workerAddress, WorkerBlockMasterClient masterClient) { mBlockDataManager = blockDataManager; mWorkerAddress = workerAddress; TachyonConf conf = WorkerContext.getConf(); mMasterClient = masterClient; mHeartbeatIntervalMs = conf.getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS); mHeartbeatTimeoutMs = conf.getInt(Constants.WORKER_BLOCK_HEARTBEAT_TIMEOUT_MS); mRunning = true; }
/** * Creates a new block worker client pool. * * @param workerAddress the worker address */ public BlockWorkerClientPool(NetAddress workerAddress) { super(ClientContext.getConf().getInt(Constants.USER_BLOCK_WORKER_CLIENT_THREADS)); mWorkerNetAddress = workerAddress; }
/** * Constructor for SessionCleaner * * @param blockDataManager the blockDataManager this checker is updating to */ public SessionCleaner(BlockDataManager blockDataManager) { mBlockDataManager = blockDataManager; mCheckIntervalMs = WorkerContext.getConf().getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS); mRunning = true; }
@Override public void close() throws IOException { int quietPeriodSecs = mTachyonConf.getInt(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_QUIET_PERIOD); int timeoutSecs = mTachyonConf.getInt(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_TIMEOUT); mChannelFuture.channel().close().awaitUninterruptibly(); mBootstrap.group().shutdownGracefully(quietPeriodSecs, timeoutSecs, TimeUnit.SECONDS); mBootstrap.childGroup().shutdownGracefully(quietPeriodSecs, timeoutSecs, TimeUnit.SECONDS); }
/** * Creates a new block master client pool. * * @param masterAddress the master address */ public BlockMasterClientPool(InetSocketAddress masterAddress) { super(ClientContext.getConf().getInt(Constants.USER_BLOCK_MASTER_CLIENT_THREADS)); mMasterAddress = masterAddress; }
/** * Creates a new file stream master client pool. * * @param masterAddress the master address */ public FileSystemMasterClientPool(InetSocketAddress masterAddress) { super(ClientContext.getConf().getInt(Constants.USER_FILE_MASTER_CLIENT_THREADS)); mMasterAddress = masterAddress; }
private void initBlockMetadataManager() throws BlockAlreadyExistsException, IOException, WorkerOutOfSpaceException { // Initialize storage tiers int totalTiers = WorkerContext.getConf().getInt(Constants.WORKER_TIERED_STORAGE_LEVEL_MAX); mAliasToTiers = new HashMap<Integer, StorageTier>(totalTiers); mTiers = new ArrayList<StorageTier>(totalTiers); for (int level = 0; level < totalTiers; level ++) { StorageTier tier = StorageTier.newStorageTier(level); mTiers.add(tier); mAliasToTiers.put(tier.getTierAlias(), tier); } }
/** * Creates a default {@link io.netty.bootstrap.ServerBootstrap} where the channel and groups are * preset. * * @param type The channel type. Current channel types supported are nio and epoll. * @return an instance of ServerBootstrap */ private ServerBootstrap createBootstrapOfType(final ChannelType type) { final ServerBootstrap boot = new ServerBootstrap(); final int bossThreadCount = mTachyonConf.getInt(Constants.WORKER_NETWORK_NETTY_BOSS_THREADS); // If number of worker threads is 0, Netty creates (#processors * 2) threads by default. final int workerThreadCount = mTachyonConf.getInt(Constants.WORKER_NETWORK_NETTY_WORKER_THREADS); final EventLoopGroup bossGroup = NettyUtils.createEventLoop(type, bossThreadCount, "data-server-boss-%d", false); final EventLoopGroup workerGroup = NettyUtils.createEventLoop(type, workerThreadCount, "data-server-worker-%d", false); final Class<? extends ServerChannel> socketChannelClass = NettyUtils.getServerChannelClass(type); boot.group(bossGroup, workerGroup).channel(socketChannelClass); return boot; } }
/** * Populate the header with information about master. So we can return to * the master from any page. * * @param request The HttpServletRequest object * @param response The HttpServletResponse object * @throws ServletException * @throws IOException */ @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { int masterWebPort = mTachyonConf.getInt(Constants.MASTER_WEB_PORT); String masterHostName = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, mTachyonConf); request.setAttribute("masterHost", masterHostName); request.setAttribute("masterPort", masterWebPort); getServletContext().getRequestDispatcher("/header.jsp").include(request, response); } }
public RawTableMaster(FileSystemMaster fileSystemMaster, Journal journal) { super(journal, Executors.newFixedThreadPool(2, ThreadFactoryUtils.build("raw-table-master-%d", true))); TachyonConf conf = MasterContext.getConf(); mMaxTableMetadataBytes = conf.getBytes(Constants.MAX_TABLE_METADATA_BYTE); mMaxColumns = conf.getInt(Constants.MAX_COLUMNS); mFileSystemMaster = fileSystemMaster; }
public void start() { mFilePersistenceService = mSyncExecutorService.submit(new HeartbeatThread(HeartbeatContext.WORKER_LINEAGE_SYNC, new LineageWorkerMasterSyncExecutor(mLineageDataManager, mLineageMasterWorkerClient), mTachyonConf.getInt(Constants.WORKER_LINEAGE_HEARTBEAT_INTERVAL_MS))); }
/** * Updates session's heartbeat. * * @param sessionId the id of the session */ public void sessionHeartbeat(long sessionId) { synchronized (mSessions) { if (mSessions.containsKey(sessionId)) { mSessions.get(sessionId).heartbeat(); } else { TachyonConf conf = WorkerContext.getConf(); int sessionTimeoutMs = conf.getInt(Constants.WORKER_SESSION_TIMEOUT_MS); mSessions.put(sessionId, new SessionInfo(sessionId, sessionTimeoutMs)); } } } }
/** * Populates key, value pairs for UI display * * @param request The HttpServletRequest object * @throws IOException */ private void populateValues(HttpServletRequest request) throws IOException { request.setAttribute("debug", Constants.DEBUG); List<WorkerInfo> workerInfos = mBlockMaster.getWorkerInfoList(); NodeInfo[] normalNodeInfos = generateOrderedNodeInfos(workerInfos); request.setAttribute("normalNodeInfos", normalNodeInfos); Set<WorkerInfo> lostWorkerInfos = mBlockMaster.getLostWorkersInfo(); NodeInfo[] failedNodeInfos = generateOrderedNodeInfos(lostWorkerInfos); request.setAttribute("failedNodeInfos", failedNodeInfos); request.setAttribute("workerWebPort", mTachyonConf.getInt(Constants.WORKER_WEB_PORT)); } }
private TachyonFS(TachyonConf tachyonConf) { super(tachyonConf); mMasterAddress = NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, tachyonConf); mZookeeperMode = mTachyonConf.getBoolean(Constants.ZOOKEEPER_ENABLED); mFSMasterClient = mCloser.register(FileSystemContext.INSTANCE.acquireMasterClient()); mBlockMasterClient = mCloser.register(BlockStoreContext.INSTANCE.acquireMasterClient()); mRawTableMasterClient = mCloser.register(new RawTableMasterClient(mMasterAddress, mTachyonConf)); mWorkerClient = mCloser.register(BlockStoreContext.INSTANCE.acquireWorkerClient()); mUserFailedSpaceRequestLimits = mTachyonConf.getInt(Constants.USER_FAILED_SPACE_REQUEST_LIMITS); String scheme = mZookeeperMode ? Constants.SCHEME_FT : Constants.SCHEME; String authority = mMasterAddress.getHostName() + ":" + mMasterAddress.getPort(); mRootUri = new TachyonURI(scheme, authority, TachyonURI.SEPARATOR); }
@Override public void heartbeat() { LOG.debug("System status checking."); TachyonConf conf = MasterContext.getConf(); int masterWorkerTimeoutMs = conf.getInt(Constants.MASTER_WORKER_TIMEOUT_MS); synchronized (mBlocks) { synchronized (mWorkers) { Iterator<MasterWorkerInfo> iter = mWorkers.iterator(); while (iter.hasNext()) { MasterWorkerInfo worker = iter.next(); final long lastUpdate = CommonUtils.getCurrentMs() - worker.getLastUpdatedTimeMs(); if (lastUpdate > masterWorkerTimeoutMs) { LOG.error("The worker {} got timed out!", worker); mLostWorkers.add(worker); iter.remove(); processLostWorker(worker); } } } } }
@Override public void start(boolean isLeader) throws IOException { super.start(isLeader); if (isLeader) { mLostWorkerDetectionService = getExecutorService().submit( new HeartbeatThread(HeartbeatContext.MASTER_LOST_WORKER_DETECTION, new LostWorkerDetectionHeartbeatExecutor(), MasterContext.getConf().getInt( Constants.MASTER_HEARTBEAT_INTERVAL_MS))); } }
public SpaceReserver(BlockDataManager blockManager) { mBlockManager = blockManager; List<Long> capOnTiers = blockManager.getStoreMeta().getCapacityBytesOnTiers(); List<Integer> aliasOnTiers = blockManager.getStoreMeta().getAliasOnTiers(); long lastTierReservedBytes = 0; for (int idx = 0; idx < aliasOnTiers.size(); idx ++) { String tierReservedSpaceProp = String.format(Constants.WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT, idx); int tierAlias = aliasOnTiers.get(idx); /** Similar to {@link BlockStoreMeta}, the alias index is the value of alias - 1 */ long reservedSpaceBytes = (long)(capOnTiers.get(tierAlias - 1) * WorkerContext.getConf() .getDouble(tierReservedSpaceProp)); mBytesToReserveOnTiers.add(new Pair<Integer, Long>(tierAlias, reservedSpaceBytes + lastTierReservedBytes)); lastTierReservedBytes += reservedSpaceBytes; } mCheckIntervalMs = WorkerContext.getConf().getInt(Constants.WORKER_TIERED_STORE_RESERVER_INTERVAL_MS); mRunning = true; }
@Override public void start(boolean isLeader) throws IOException { super.start(isLeader); if (isLeader) { mCheckpointExecutionService = getExecutorService().submit( new HeartbeatThread(HeartbeatContext.MASTER_CHECKPOINT_SCHEDULING, new CheckpointSchedulingExcecutor(this), mTachyonConf .getInt(Constants.MASTER_LINEAGE_CHECKPOINT_INTERVAL_MS))); mRecomputeExecutionService = getExecutorService().submit( new HeartbeatThread(HeartbeatContext.MASTER_FILE_RECOMPUTATION, new RecomputeExecutor(new RecomputePlanner(mLineageStore, mFileSystemMaster), mFileSystemMaster), mTachyonConf .getInt(Constants.MASTER_LINEAGE_RECOMPUTE_INTERVAL_MS))); } }