/** * Configures and starts master. * * @throws IOException when the operation fails */ public void startMaster() throws IOException { mMasterConf = MasterContext.getConf(); mMasterConf.set(Constants.IN_TEST_MODE, "true"); mMasterConf.set(Constants.TACHYON_HOME, mTachyonHome); mMasterConf.set(Constants.USER_QUOTA_UNIT_BYTES, Integer.toString(mQuotaUnitBytes)); mMasterConf.set(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT, Integer.toString(mUserBlockSize)); mMasterConf.set(Constants.USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES, Integer.toString(64)); mMasterConf.set(Constants.MASTER_HOSTNAME, mLocalhostName); mMasterConf.set(Constants.MASTER_PORT, Integer.toString(0)); mMasterConf.set(Constants.MASTER_WEB_PORT, Integer.toString(0)); mMasterConf.set(Constants.MASTER_TTLCHECKER_INTERVAL_MS, Integer.toString(1000)); // default write type becomes MUST_CACHE, set this value to CACHE_THROUGH for tests. // default tachyon storage is STORE, and under storage is SYNC_PERSIST for tests. // TODO(binfan): eliminate this setting after updating integration tests mMasterConf.set(Constants.USER_FILE_WRITE_TYPE_DEFAULT, "CACHE_THROUGH"); mMaster = LocalTachyonMaster.create(mTachyonHome); mMaster.start(); }
/** * Creates a <code>TachyonFS</code> handler for the given hostname, port, and Zookeeper mode. * * @param masterHost master host details * @param masterPort port master listens on * @param zkMode use zookeeper * @return the corresponding TachyonFS handler */ public static synchronized TachyonFS get(String masterHost, int masterPort, boolean zkMode) { TachyonConf tachyonConf = ClientContext.getConf(); tachyonConf.set(Constants.MASTER_HOSTNAME, masterHost); tachyonConf.set(Constants.MASTER_PORT, Integer.toString(masterPort)); tachyonConf.set(Constants.ZOOKEEPER_ENABLED, Boolean.toString(zkMode)); return get(tachyonConf); }
mWorkerConf = WorkerContext.getConf(); mWorkerConf.merge(mMasterConf); mWorkerConf.set(Constants.WORKER_PORT, Integer.toString(0)); mWorkerConf.set(Constants.WORKER_DATA_PORT, Integer.toString(0)); mWorkerConf.set(Constants.WORKER_WEB_PORT, Integer.toString(0)); mWorkerConf.set(Constants.WORKER_DATA_FOLDER, mWorkerDataFolder); mWorkerConf.set(Constants.WORKER_MEMORY_SIZE, Long.toString(mWorkerCapacityBytes)); mWorkerConf.set(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS, Integer.toString(15)); mWorkerConf.set(Constants.WORKER_WORKER_BLOCK_THREADS_MIN, Integer.toString(1)); mWorkerConf.set(Constants.WORKER_WORKER_BLOCK_THREADS_MAX, Integer.toString(2048)); mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_WORKER_THREADS, Integer.toString(2)); mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_QUIET_PERIOD, Integer.toString(0)); mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_TIMEOUT, Integer.toString(0)); mWorkerConf.set(Constants.NETWORK_HOST_RESOLUTION_TIMEOUT_MS, Integer.toString(250)); mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORE_LEVEL_ALIAS_FORMAT, 0), "MEM"); mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT, 0), ramdiskPath); mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA_FORMAT, 0), Long.toString(mWorkerCapacityBytes)); UnderFileSystemUtils.mkdirIfNotExists(ramdiskPath, mWorkerConf); UnderFileSystemUtils.mkdirIfNotExists(newPath, mWorkerConf); mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT, level), Joiner.on(',').join(newPaths));
/** * This method adds AWS credentials from system properties to the Tachyon Conf if they are not * already present. * @param tachyonConf the conf to check and add credentials to * @return true if both access and secret key are present, false otherwise */ private boolean addAndCheckAWSCredentials(TachyonConf tachyonConf) { String accessKeyConf = Constants.S3_ACCESS_KEY; if (System.getProperty(accessKeyConf) != null && tachyonConf.get(accessKeyConf) == null) { tachyonConf.set(accessKeyConf, System.getProperty(accessKeyConf)); } String secretKeyConf = Constants.S3_SECRET_KEY; if (System.getProperty(secretKeyConf) != null && tachyonConf.get(secretKeyConf) == null) { tachyonConf.set(secretKeyConf, System.getProperty(secretKeyConf)); } return tachyonConf.get(accessKeyConf) != null && tachyonConf.get(secretKeyConf) != null; } }
mMasterConf.set(Constants.IN_TEST_MODE, "true"); mMasterConf.set(Constants.TACHYON_HOME, mTachyonHome); mMasterConf.set(Constants.ZOOKEEPER_ENABLED, "true"); mMasterConf.set(Constants.MASTER_HOSTNAME, mHostname); mMasterConf.set(Constants.MASTER_BIND_HOST, mHostname); mMasterConf.set(Constants.MASTER_PORT, "0"); mMasterConf.set(Constants.MASTER_WEB_BIND_HOST, mHostname); mMasterConf.set(Constants.MASTER_WEB_PORT, "0"); mMasterConf.set(Constants.ZOOKEEPER_ADDRESS, mCuratorServer.getConnectString()); mMasterConf.set(Constants.ZOOKEEPER_ELECTION_PATH, "/election"); mMasterConf.set(Constants.ZOOKEEPER_LEADER_PATH, "/leader"); mMasterConf.set(Constants.USER_QUOTA_UNIT_BYTES, "10000"); mMasterConf.set(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT, Integer.toString(mUserBlockSize)); mMasterConf.set(Constants.MASTER_TTLCHECKER_INTERVAL_MS, Integer.toString(1000)); mMasterConf.set(Constants.NETWORK_HOST_RESOLUTION_TIMEOUT_MS, "250"); mMasterConf.set(Constants.MASTER_PORT, "0"); mMasterConf.set(Constants.MASTER_PORT, getMasterPort() + ""); mWorkerConf.set(Constants.WORKER_DATA_FOLDER, mWorkerDataFolder); mWorkerConf.set(Constants.WORKER_MEMORY_SIZE, mWorkerCapacityBytes + ""); mWorkerConf.set(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS, 15 + ""); mWorkerConf.set(Constants.WORKER_TIERED_STORAGE_LEVEL_MAX, Integer.toString(maxLevel)); mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORE_LEVEL_ALIAS_FORMAT, 0), "MEM"); mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT, 0), mTachyonHome + "/ramdisk");
/** * Creates a <code>TachyonFS</code> handler for the given Tachyon URI and configuration. * * @param tachyonURI a Tachyon URI to indicate master address. e.g., tachyon://localhost:19998, * tachyon://localhost:19998/ab/c.txt * @param tachyonConf The TachyonConf instance. * @return the corresponding TachyonFS handler */ public static synchronized TachyonFS get(final TachyonURI tachyonURI, TachyonConf tachyonConf) { Preconditions.checkArgument(tachyonConf != null, "TachyonConf cannot be null."); Preconditions.checkArgument(tachyonURI != null, "Tachyon URI cannot be null. Use " + Constants.HEADER + "host:port/ ," + Constants.HEADER_FT + "host:port/."); String scheme = tachyonURI.getScheme(); Preconditions.checkNotNull(scheme, "Tachyon scheme cannot be null. Use " + Constants.SCHEME + " or " + Constants.SCHEME_FT + "."); Preconditions.checkNotNull(tachyonURI.getHost(), "Tachyon hostname cannot be null."); Preconditions.checkState(tachyonURI.getPort() != -1, "Tachyon URI must have a port number."); Preconditions.checkState( (Constants.SCHEME.equals(scheme) || Constants.SCHEME_FT.equals(scheme)), "Tachyon scheme must be either " + Constants.SCHEME + " or " + Constants.SCHEME_FT + "."); boolean useZookeeper = scheme.equals(Constants.SCHEME_FT); tachyonConf.set(Constants.ZOOKEEPER_ENABLED, Boolean.toString(useZookeeper)); tachyonConf.set(Constants.MASTER_HOSTNAME, tachyonURI.getHost()); tachyonConf.set(Constants.MASTER_PORT, Integer.toString(tachyonURI.getPort())); return get(tachyonConf); }
/** * Starts the web server. */ public void startWebServer() { try { mServer.getConnectors()[0].close(); mServer.getConnectors()[0].open(); mServer.start(); if (mAddress.getPort() == 0) { int webPort = mServer.getConnectors()[0].getLocalPort(); mAddress = new InetSocketAddress(mAddress.getHostName(), webPort); // reset web service port mTachyonConf.set(mService.getPortKey(), Integer.toString(webPort)); } LOG.info(mService.getServiceName() + " started @ " + mAddress); } catch (Exception e) { throw Throwables.propagate(e); } } }
/** * {@inheritDoc} * * Sets up a lazy connection to Tachyon through mTFS. */ @Override public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); LOG.info("initialize(" + uri + ", " + conf + "). Connecting to Tachyon: " + uri.toString()); Utils.addS3Credentials(conf); setConf(conf); mTachyonHeader = getScheme() + "://" + uri.getHost() + ":" + uri.getPort(); // Set the statistics member. Use mStatistics instead of the parent class's variable. mStatistics = statistics; // Load TachyonConf if any and merge to the one in TachyonFS TachyonConf siteConf = ConfUtils.loadFromHadoopConfiguration(conf); if (siteConf != null) { mTachyonConf.merge(siteConf); } mTachyonConf.set(Constants.MASTER_HOSTNAME, uri.getHost()); mTachyonConf.set(Constants.MASTER_PORT, Integer.toString(uri.getPort())); mTachyonConf.set(Constants.ZOOKEEPER_ENABLED, Boolean.toString(isZookeeperMode())); mTFS = TachyonFS.get(mTachyonConf); mUri = URI.create(mTachyonHeader); mUnderFSAddress = mTFS.getUfsAddress(); LOG.info(mTachyonHeader + " " + mUri + " " + mUnderFSAddress); }
tachyonConf); tachyonConf.set(Constants.MASTER_JOURNAL_FOLDER, mJournalFolder); tachyonConf.set(Constants.UNDERFS_ADDRESS, mUfsDirectory); tachyonConf.set(Constants.MASTER_WORKER_THREADS_MIN, "1"); tachyonConf.set(Constants.MASTER_WORKER_THREADS_MAX, "100"); tachyonConf.set(Constants.MASTER_RETRY_COUNT, "3"); tachyonConf.set(Constants.NETWORK_HOST_RESOLUTION_TIMEOUT_MS, "250"); tachyonConf.set(Constants.WEB_THREAD_COUNT, "1"); tachyonConf.set(Constants.WEB_RESOURCES, PathUtils.concatPath(System.getProperty("user.dir"), "../servers/src/main/webapp")); tachyonConf.set(Constants.MASTER_PORT, Integer.toString(getRPCLocalPort()));
@Override public Boolean call() throws Exception { TachyonConf tachyonConf = ClientContext.getConf(); tachyonConf.set(Constants.MASTER_HOSTNAME, mMasterLocation.getHost()); tachyonConf.set(Constants.MASTER_PORT, Integer.toString(mMasterLocation.getPort())); ClientContext.reset(tachyonConf); TachyonFileSystem tFS = TachyonFileSystem.TachyonFileSystemFactory.get(); write(tFS, mFilePath, mWriteType, mDeleteIfExists, mLength); return read(tFS, mFilePath, mReadType); }
@Override public Boolean call() throws Exception { TachyonConf tachyonConf = ClientContext.getConf(); tachyonConf.set(Constants.MASTER_HOSTNAME, mMasterLocation.getHost()); tachyonConf.set(Constants.MASTER_PORT, Integer.toString(mMasterLocation.getPort())); ClientContext.reset(tachyonConf); TachyonFileSystem tFS = TachyonFileSystem.TachyonFileSystemFactory.get(); long fileId = createFile(tFS); writeFile(fileId); return readFile(tFS, fileId); }
/** * Creates a new local tachyon master with a isolated home and port. * * @throws IOException when unable to do file operation or listen on port * @return an instance of Tachyon master */ public static LocalTachyonMaster create() throws IOException { final String tachyonHome = uniquePath(); TachyonConf tachyonConf = MasterContext.getConf(); UnderFileSystemUtils.deleteDir(tachyonHome, tachyonConf); UnderFileSystemUtils.mkdirIfNotExists(tachyonHome, tachyonConf); // Update Tachyon home in the passed TachyonConf instance. tachyonConf.set(Constants.TACHYON_HOME, tachyonHome); return new LocalTachyonMaster(tachyonHome); }
mPort = NetworkAddressUtils.getThriftPort(mTServerSocket); conf.set(Constants.MASTER_PORT, Integer.toString(mPort)); mMasterAddress = NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, conf);
mBlockDataManager, mTachyonConf); mTachyonConf.set(Constants.WORKER_DATA_PORT, Integer.toString(mDataServer.getPort())); mPort = NetworkAddressUtils.getThriftPort(mThriftServerSocket); mTachyonConf.set(Constants.WORKER_PORT, Integer.toString(mPort)); mThriftServer = createThriftServer(); mWorkerNetAddress =