/** * Creates a new instance of {@link Address} from proto representation. * * @param address the proto representation of a master address * @return the instance */ public static Address fromProto(alluxio.grpc.NetAddress address) { return new Address(address); }
/** * Re-register a lost worker or complete registration after getting a worker id. * * @param workerId the worker id to register */ @Nullable private MasterWorkerInfo registerWorkerInternal(long workerId) { for (IndexedSet<MasterWorkerInfo> workers: Arrays.asList(mTempWorkers, mLostWorkers)) { MasterWorkerInfo worker = workers.getFirstByField(ID_INDEX, workerId); if (worker == null) { continue; } synchronized (worker) { worker.updateLastUpdatedTimeMs(); mWorkers.add(worker); workers.remove(worker); if (workers == mLostWorkers) { for (Consumer<Address> function : mLostWorkerFoundListeners) { function.accept(new Address(worker.getWorkerAddress().getHost(), worker.getWorkerAddress().getRpcPort())); } LOG.warn("A lost worker {} has requested its old id {}.", worker.getWorkerAddress(), worker.getId()); } } return worker; } return null; }
@Override public void heartbeat() { long masterWorkerTimeoutMs = ServerConfiguration.getMs(PropertyKey.MASTER_WORKER_TIMEOUT_MS); for (MasterWorkerInfo worker : mWorkers) { synchronized (worker) { final long lastUpdate = mClock.millis() - worker.getLastUpdatedTimeMs(); if (lastUpdate > masterWorkerTimeoutMs) { LOG.error("The worker {}({}) timed out after {}ms without a heartbeat!", worker.getId(), worker.getWorkerAddress(), lastUpdate); mLostWorkers.add(worker); mWorkers.remove(worker); WorkerNetAddress workerAddress = worker.getWorkerAddress(); for (Consumer<Address> function : mWorkerLostListeners) { function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort())); } processWorkerRemovedBlocks(worker, worker.getBlocks()); } } } }
for (BiConsumer<Address, List<ConfigProperty>> function : mWorkerRegisteredListeners) { WorkerNetAddress workerAddress = worker.getWorkerAddress(); function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()), options.getConfigsList());
@Test public void registerNewConfUnknownProperty() { Address testAddress = new Address("test", 0); ServerConfigurationStore configStore = new ServerConfigurationStore(); configStore.registerNewConf(testAddress, Arrays.asList(ConfigProperty.newBuilder().setName("unknown.property").build() )); Map<Address, List<ConfigRecord>> confMap = configStore.getConfMap(); assertTrue(confMap.containsKey(testAddress)); assertEquals("unknown.property", confMap.get(testAddress).get(0).getKey().getName()); }
@Before public void before() { PropertyKey keyEnforce = PropertyKey.ZOOKEEPER_ELECTION_PATH; PropertyKey keyWarn = PropertyKey.WORKER_FREE_SPACE_TIMEOUT; mConfigListOne = Arrays.asList( ConfigProperty.newBuilder().setName(keyEnforce.getName()).setSource("Test") .setValue("Value").build(), ConfigProperty.newBuilder().setName(keyWarn.getName()).setSource("Test").setValue("Value2") .build()); mConfigListTwo = Arrays.asList( ConfigProperty.newBuilder().setName(keyEnforce.getName()).setSource("Test") .setValue("Value3").build(), ConfigProperty.newBuilder().setName(keyWarn.getName()).setSource("Test").setValue("Value4") .build()); Random random = new Random(); mAddressOne = new Address(RandomStringUtils.randomAlphanumeric(10), random.nextInt()); mAddressTwo = new Address(RandomStringUtils.randomAlphanumeric(10), random.nextInt()); }
/** * Creates a new instance of {@link DefaultMetaMaster}. * * @param blockMaster a block master handle * @param masterContext the context for Alluxio master * @param executorServiceFactory a factory for creating the executor service to use for running * maintenance threads */ DefaultMetaMaster(BlockMaster blockMaster, CoreMasterContext masterContext, ExecutorServiceFactory executorServiceFactory) { super(masterContext, new SystemClock(), executorServiceFactory); mMasterAddress = new Address().setHost(ServerConfiguration.getOrDefault(PropertyKey.MASTER_HOSTNAME, "localhost")) .setRpcPort(mPort); mBlockMaster = blockMaster; mBlockMaster.registerLostWorkerFoundListener(mWorkerConfigStore::lostNodeFound); mBlockMaster.registerWorkerLostListener(mWorkerConfigStore::handleNodeLost); mBlockMaster.registerNewWorkerConfListener(mWorkerConfigStore::registerNewConf); if (URIUtils.isLocalFilesystem(ServerConfiguration .get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS))) { mUfs = UnderFileSystem.Factory .create("/", UnderFileSystemConfiguration.defaults()); } else { mUfs = UnderFileSystem.Factory.createForRoot(ServerConfiguration.global()); } }
/** * Creates a new instance of {@link Address} from thrift representation. * * @param address the thrift representation of a master address * @return the instance */ public static Address fromThrift(alluxio.thrift.MasterNetAddress address) { return new Address(address); }
/** * Re-register a lost worker or complete registration after getting a worker id. * * @param workerId the worker id to register */ @Nullable private MasterWorkerInfo registerWorkerInternal(long workerId) { for (IndexedSet<MasterWorkerInfo> workers: Arrays.asList(mTempWorkers, mLostWorkers)) { MasterWorkerInfo worker = workers.getFirstByField(ID_INDEX, workerId); if (worker == null) { continue; } synchronized (worker) { worker.updateLastUpdatedTimeMs(); mWorkers.add(worker); workers.remove(worker); if (workers == mLostWorkers) { for (Consumer<Address> function : mLostWorkerFoundListeners) { function.accept(new Address(worker.getWorkerAddress().getHost(), worker.getWorkerAddress().getRpcPort())); } LOG.warn("A lost worker {} has requested its old id {}.", worker.getWorkerAddress(), worker.getId()); } } return worker; } return null; }
for (BiConsumer<Address, List<ConfigProperty>> function : mWorkerRegisteredListeners) { WorkerNetAddress workerAddress = worker.getWorkerAddress(); function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()), wireConfigList);
@Override public void heartbeat() { long masterWorkerTimeoutMs = Configuration.getMs(PropertyKey.MASTER_WORKER_TIMEOUT_MS); for (MasterWorkerInfo worker : mWorkers) { synchronized (worker) { final long lastUpdate = mClock.millis() - worker.getLastUpdatedTimeMs(); if (lastUpdate > masterWorkerTimeoutMs) { LOG.error("The worker {}({}) timed out after {}ms without a heartbeat!", worker.getId(), worker.getWorkerAddress(), lastUpdate); mLostWorkers.add(worker); mWorkers.remove(worker); WorkerNetAddress workerAddress = worker.getWorkerAddress(); for (Consumer<Address> function : mWorkerLostListeners) { function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort())); } processWorkerRemovedBlocks(worker, worker.getBlocks()); } } } }
/** * Creates a new instance of {@link DefaultMetaMaster}. * * @param blockMaster a block master handle * @param masterContext the context for Alluxio master * @param executorServiceFactory a factory for creating the executor service to use for running * maintenance threads */ DefaultMetaMaster(BlockMaster blockMaster, MasterContext masterContext, ExecutorServiceFactory executorServiceFactory) { super(masterContext, new SystemClock(), executorServiceFactory); mSafeModeManager = masterContext.getSafeModeManager(); mStartTimeMs = masterContext.getStartTimeMs(); mMasterAddress = new Address().setHost(Configuration.getOrDefault(PropertyKey.MASTER_HOSTNAME, "localhost")) .setRpcPort(masterContext.getPort()); mBlockMaster = blockMaster; mBlockMaster.registerLostWorkerFoundListener(mWorkerConfigStore::lostNodeFound); mBlockMaster.registerWorkerLostListener(mWorkerConfigStore::handleNodeLost); mBlockMaster.registerNewWorkerConfListener(mWorkerConfigStore::registerNewConf); }