public static synchronized TachyonLineage get() { if (sTachyonLineage == null) { if (!ClientContext.getConf().getBoolean(Constants.USER_LINEAGE_ENABLED)) { throw new IllegalStateException("Lineage is not enabled in the configuration."); } sTachyonLineage = new TachyonLineage(); } return sTachyonLineage; }
private TachyonFS(TachyonConf tachyonConf) { super(tachyonConf); mMasterAddress = NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, tachyonConf); mZookeeperMode = mTachyonConf.getBoolean(Constants.ZOOKEEPER_ENABLED); mFSMasterClient = mCloser.register(FileSystemContext.INSTANCE.acquireMasterClient()); mBlockMasterClient = mCloser.register(BlockStoreContext.INSTANCE.acquireMasterClient()); mRawTableMasterClient = mCloser.register(new RawTableMasterClient(mMasterAddress, mTachyonConf)); mWorkerClient = mCloser.register(BlockStoreContext.INSTANCE.acquireWorkerClient()); mUserFailedSpaceRequestLimits = mTachyonConf.getInt(Constants.USER_FAILED_SPACE_REQUEST_LIMITS); String scheme = mZookeeperMode ? Constants.SCHEME_FT : Constants.SCHEME; String authority = mMasterAddress.getHostName() + ":" + mMasterAddress.getPort(); mRootUri = new TachyonURI(scheme, authority, TachyonURI.SEPARATOR); }
/** * @return {@link TachyonMasterFaultTolerant} if tachyonConf is set to use zookeeper, otherwise, * return {@link TachyonMaster}. */ public static TachyonMaster createMaster() { if (MasterContext.getConf().getBoolean(Constants.ZOOKEEPER_ENABLED)) { return new TachyonMasterFaultTolerant(); } return new TachyonMaster(); } }
public static synchronized TachyonFileSystem get() { if (sTachyonFileSystem == null) { boolean enableLineage = ClientContext.getConf().getBoolean(Constants.USER_LINEAGE_ENABLED); sTachyonFileSystem = enableLineage ? TachyonLineageFileSystem.get() : new TachyonFileSystem(); } return sTachyonFileSystem; } }
/** * Validates the path, verifying that it contains the <code>Constants.HEADER </code> or * <code>Constants.HEADER_FT</code> and a hostname:port specified. * * @param path The path to be verified. * @param tachyonConf The instance of {@link tachyon.conf.TachyonConf} to be used. * @return the verified path in a form like tachyon://host:port/dir. If only the "/dir" or "dir" * part is provided, the host and port are retrieved from property, * tachyon.master.hostname and tachyon.master.port, respectively. * @throws IOException if the given path is not valid. */ public static String validatePath(String path, TachyonConf tachyonConf) throws IOException { if (path.startsWith(Constants.HEADER) || path.startsWith(Constants.HEADER_FT)) { if (!path.contains(":")) { throw new IOException("Invalid Path: " + path + ". Use " + Constants.HEADER + "host:port/ ," + Constants.HEADER_FT + "host:port/" + " , or /file"); } else { return path; } } else { String hostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, tachyonConf); int port = tachyonConf.getInt(Constants.MASTER_PORT); if (tachyonConf.getBoolean(Constants.ZOOKEEPER_ENABLED)) { return PathUtils.concatPath(Constants.HEADER_FT + hostname + ":" + port, path); } return PathUtils.concatPath(Constants.HEADER + hostname + ":" + port, path); } }
public TachyonMasterFaultTolerant() { super(); TachyonConf conf = MasterContext.getConf(); Preconditions.checkArgument(conf.getBoolean(Constants.ZOOKEEPER_ENABLED)); // Set up zookeeper specific functionality. try { // InetSocketAddress.toString causes test issues, so build the string by hand String zkName = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, conf) + ":" + getMasterAddress().getPort(); String zkAddress = conf.get(Constants.ZOOKEEPER_ADDRESS); String zkElectionPath = conf.get(Constants.ZOOKEEPER_ELECTION_PATH); String zkLeaderPath = conf.get(Constants.ZOOKEEPER_LEADER_PATH); mLeaderSelectorClient = new LeaderSelectorClient(zkAddress, zkElectionPath, zkLeaderPath, zkName); } catch (Exception e) { LOG.error(e.getMessage(), e); throw Throwables.propagate(e); } }
public S3UnderFileSystem(String bucketName, TachyonConf tachyonConf) throws ServiceException { super(tachyonConf); Preconditions.checkArgument(tachyonConf.containsKey(Constants.S3_ACCESS_KEY), "Property " + Constants.S3_ACCESS_KEY + " is required to connect to S3"); Preconditions.checkArgument(tachyonConf.containsKey(Constants.S3_SECRET_KEY), "Property " + Constants.S3_SECRET_KEY + " is required to connect to S3"); AWSCredentials awsCredentials = new AWSCredentials(tachyonConf.get(Constants.S3_ACCESS_KEY), tachyonConf.get( Constants.S3_SECRET_KEY)); mBucketName = bucketName; Jets3tProperties props = new Jets3tProperties(); if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) { props.setProperty("httpclient.proxy-autodetect", "false"); props.setProperty("httpclient.proxy-host", tachyonConf.get(Constants.UNDERFS_S3_PROXY_HOST)); props.setProperty("httpclient.proxy-port", tachyonConf.get(Constants.UNDERFS_S3_PROXY_PORT)); } if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) { props.setProperty("s3service.https-only", Boolean.toString(tachyonConf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY))); } LOG.debug("Initializing S3 underFs with properties: " + props.getProperties()); mClient = new RestS3Service(awsCredentials, null, null, props); mBucketPrefix = Constants.HEADER_S3N + mBucketName + PATH_SEPARATOR; }
if (mTachyonConf.getBoolean(Constants.WORKER_TIERED_STORE_RESERVER_ENABLED)) { mSpaceReserver = new SpaceReserver(mBlockDataManager);