/** * Replaces default key with user provided key. * * @param conf configuration to replace the key in * @param tachyonConf Tachyon configuration with the key * @param key the key to replace */ public static void addKey(Configuration conf, TachyonConf tachyonConf, String key) { if (System.getProperty(key) != null) { conf.set(key, System.getProperty(key)); } else if (tachyonConf.get(key) != null) { conf.set(key, tachyonConf.get(key)); } }
private SortedSet<Pair<String, String>> getSortedProperties() { TreeSet<Pair<String, String>> rtn = new TreeSet<Pair<String, String>>(); for (Map.Entry<Object, Object> entry : mTachyonConf.getInternalProperties().entrySet()) { String key = entry.getKey().toString(); if (key.startsWith(TACHYON_CONF_PREFIX) && !TACHYON_CONF_EXCLUDES.contains(key)) { rtn.add(new ImmutablePair<String, String>(key, mTachyonConf.get(key))); } } return rtn; } }
public TachyonMasterFaultTolerant() { super(); TachyonConf conf = MasterContext.getConf(); Preconditions.checkArgument(conf.getBoolean(Constants.ZOOKEEPER_ENABLED)); // Set up zookeeper specific functionality. try { // InetSocketAddress.toString causes test issues, so build the string by hand String zkName = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, conf) + ":" + getMasterAddress().getPort(); String zkAddress = conf.get(Constants.ZOOKEEPER_ADDRESS); String zkElectionPath = conf.get(Constants.ZOOKEEPER_ELECTION_PATH); String zkLeaderPath = conf.get(Constants.ZOOKEEPER_LEADER_PATH); mLeaderSelectorClient = new LeaderSelectorClient(zkAddress, zkElectionPath, zkLeaderPath, zkName); } catch (Exception e) { LOG.error(e.getMessage(), e); throw Throwables.propagate(e); } }
/** * Initializes the client context singleton with a given conf. */ public static synchronized void reset(TachyonConf conf) { sTachyonConf = conf; String masterHostname = Preconditions.checkNotNull(sTachyonConf.get(Constants.MASTER_HOSTNAME)); int masterPort = sTachyonConf.getInt(Constants.MASTER_PORT); sMasterAddress = new InetSocketAddress(masterHostname, masterPort); sClientMetrics = new ClientMetrics(); sRandom = new Random(); if (sExecutorService != null) { sExecutorService.shutdown(); } sExecutorService = Executors.newFixedThreadPool( sTachyonConf.getInt(Constants.USER_BLOCK_WORKER_CLIENT_THREADS), ThreadFactoryUtils.build("block-worker-heartbeat-%d", true)); // We must set sInitialized to true before resetting BlockStoreContext and FileSystemContext // as they need ClientContext initialized. sInitialized = true; BlockStoreContext.INSTANCE.reset(); FileSystemContext.INSTANCE.reset(); }
@Override protected void prepareConfiguration(String path, TachyonConf tachyonConf, Configuration config) { if (path.startsWith(SCHEME)) { // Configure for Gluster FS config.set("fs.glusterfs.impl", tachyonConf.get(Constants.UNDERFS_GLUSTERFS_IMPL)); config.set("mapred.system.dir", tachyonConf.get(Constants.UNDERFS_GLUSTERFS_MR_DIR)); config .set("fs.glusterfs.volumes", tachyonConf.get(Constants.UNDERFS_GLUSTERFS_VOLUMES)); config.set( "fs.glusterfs.volume.fuse." + tachyonConf.get(Constants.UNDERFS_GLUSTERFS_VOLUMES), tachyonConf.get(Constants.UNDERFS_GLUSTERFS_MOUNTS)); } else { // If not Gluster FS fall back to default HDFS behaviour // This should only happen if someone creates an instance of this directly rather than via the // registry and factory which enforces the GlusterFS prefix being present. super.prepareConfiguration(path, tachyonConf, config); } }
public S3UnderFileSystem(String bucketName, TachyonConf tachyonConf) throws ServiceException { super(tachyonConf); Preconditions.checkArgument(tachyonConf.containsKey(Constants.S3_ACCESS_KEY), "Property " + Constants.S3_ACCESS_KEY + " is required to connect to S3"); Preconditions.checkArgument(tachyonConf.containsKey(Constants.S3_SECRET_KEY), "Property " + Constants.S3_SECRET_KEY + " is required to connect to S3"); AWSCredentials awsCredentials = new AWSCredentials(tachyonConf.get(Constants.S3_ACCESS_KEY), tachyonConf.get( Constants.S3_SECRET_KEY)); mBucketName = bucketName; Jets3tProperties props = new Jets3tProperties(); if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) { props.setProperty("httpclient.proxy-autodetect", "false"); props.setProperty("httpclient.proxy-host", tachyonConf.get(Constants.UNDERFS_S3_PROXY_HOST)); props.setProperty("httpclient.proxy-port", tachyonConf.get(Constants.UNDERFS_S3_PROXY_PORT)); } if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) { props.setProperty("s3service.https-only", Boolean.toString(tachyonConf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY))); } LOG.debug("Initializing S3 underFs with properties: " + props.getProperties()); mClient = new RestS3Service(awsCredentials, null, null, props); mBucketPrefix = Constants.HEADER_S3N + mBucketName + PATH_SEPARATOR; }
/** * @return the ufs address for this master */ public String getUfsAddress() { return MasterContext.getConf().get(Constants.UNDERFS_ADDRESS); }
/** * This method adds AWS credentials from system properties to the Tachyon Conf if they are not * already present. * @param tachyonConf the conf to check and add credentials to * @return true if both access and secret key are present, false otherwise */ private boolean addAndCheckAWSCredentials(TachyonConf tachyonConf) { String accessKeyConf = Constants.S3_ACCESS_KEY; if (System.getProperty(accessKeyConf) != null && tachyonConf.get(accessKeyConf) == null) { tachyonConf.set(accessKeyConf, System.getProperty(accessKeyConf)); } String secretKeyConf = Constants.S3_SECRET_KEY; if (System.getProperty(secretKeyConf) != null && tachyonConf.get(secretKeyConf) == null) { tachyonConf.set(secretKeyConf, System.getProperty(secretKeyConf)); } return tachyonConf.get(accessKeyConf) != null && tachyonConf.get(secretKeyConf) != null; } }
/** * Start Tachyon Master by executing the launch script. */ private static void startMaster() { String startMasterCommand = new TachyonConf().get(Constants.TACHYON_HOME) + "/bin/tachyon-start.sh master"; try { Runtime.getRuntime().exec(startMasterCommand).waitFor(); CommonUtils.sleepMs(LOG, 1000); } catch (Exception e) { LOG.error("Error when starting Master", e); } }
@Override public void connectFromMaster(TachyonConf conf, String host) throws IOException { if (!conf.containsKey(Constants.MASTER_KEYTAB_KEY) || !conf.containsKey(Constants.MASTER_PRINCIPAL_KEY)) { return; } String masterKeytab = conf.get(Constants.MASTER_KEYTAB_KEY); String masterPrincipal = conf.get(Constants.MASTER_PRINCIPAL_KEY); login(Constants.MASTER_KEYTAB_KEY, masterKeytab, Constants.MASTER_PRINCIPAL_KEY, masterPrincipal, host); }
public LineageDataManager(BlockDataManager blockDataManager) { mBlockDataManager = Preconditions.checkNotNull(blockDataManager); mPersistedFiles = Lists.newArrayList(); mTachyonConf = WorkerContext.getConf(); // Create Under FileSystem Client String ufsAddress = mTachyonConf.get(Constants.UNDERFS_ADDRESS); mUfs = UnderFileSystem.get(ufsAddress, mTachyonConf); }
@Override public void connectFromWorker(TachyonConf conf, String host) throws IOException { if (!conf.containsKey(Constants.WORKER_KEYTAB_KEY) || !conf.containsKey(Constants.WORKER_PRINCIPAL_KEY)) { return; } String workerKeytab = conf.get(Constants.WORKER_KEYTAB_KEY); String workerPrincipal = conf.get(Constants.WORKER_PRINCIPAL_KEY); login(Constants.WORKER_KEYTAB_KEY, workerKeytab, Constants.WORKER_PRINCIPAL_KEY, workerPrincipal, host); }
/** * Stop the current Tachyon cluster. This is used for preparation and clean up. * To crash the Master, use <code>killMaster</code>. */ private static void stopCluster() { String stopClusterCommand = new TachyonConf().get(Constants.TACHYON_HOME) + "/bin/tachyon-stop.sh"; try { Runtime.getRuntime().exec(stopClusterCommand).waitFor(); CommonUtils.sleepMs(LOG, 1000); } catch (Exception e) { LOG.error("Error when stop Tachyon cluster", e); } } }
private void initStorageTier() throws BlockAlreadyExistsException, IOException, WorkerOutOfSpaceException { String workerDataFolder = WorkerContext.getConf().get(Constants.WORKER_DATA_FOLDER); String tierDirPathConf = String.format(Constants.WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT, mTierLevel); String[] dirPaths = WorkerContext.getConf().get(tierDirPathConf).split(","); // Add the worker data folder path after each storage directory, the final path will be like // /mnt/ramdisk/tachyonworker for (int i = 0; i < dirPaths.length; i ++) { dirPaths[i] = PathUtils.concatPath(dirPaths[i].trim(), workerDataFolder); } String tierDirCapacityConf = String.format(Constants.WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA_FORMAT, mTierLevel); String[] dirQuotas = WorkerContext.getConf().get(tierDirCapacityConf).split(","); mDirs = new ArrayList<StorageDir>(dirPaths.length); long totalCapacity = 0; for (int i = 0; i < dirPaths.length; i ++) { int index = i >= dirQuotas.length ? dirQuotas.length - 1 : i; long capacity = FormatUtils.parseSpaceSize(dirQuotas[index]); totalCapacity += capacity; mDirs.add(StorageDir.newStorageDir(this, i, capacity, dirPaths[i])); } mCapacityBytes = totalCapacity; }
/** * Creates a <code>MetricsSystem</code> using the default metrics config. * * @param instance the instance name * @param tachyonConf the {@link TachyonConf} instance for configuration properties. */ public MetricsSystem(String instance, TachyonConf tachyonConf) { mInstance = instance; mTachyonConf = tachyonConf; String metricsConfFile = null; if (mTachyonConf.containsKey(Constants.METRICS_CONF_FILE)) { metricsConfFile = mTachyonConf.get(Constants.METRICS_CONF_FILE); } mMetricsConfig = new MetricsConfig(metricsConfFile); }
@Override public Long getValue() { long ret = 0L; try { String ufsDataFolder = MasterContext.getConf().get(Constants.UNDERFS_ADDRESS); UnderFileSystem ufs = UnderFileSystem.get(ufsDataFolder, MasterContext.getConf()); ret = ufs.getSpace(ufsDataFolder, UnderFileSystem.SpaceType.SPACE_TOTAL); } catch (IOException e) { LOG.error(e.getMessage(), e); } return ret; } });
@Override public Long getValue() { long ret = 0L; try { String ufsDataFolder = MasterContext.getConf().get(Constants.UNDERFS_ADDRESS); UnderFileSystem ufs = UnderFileSystem.get(ufsDataFolder, MasterContext.getConf()); ret = ufs.getSpace(ufsDataFolder, UnderFileSystem.SpaceType.SPACE_USED); } catch (IOException e) { LOG.error(e.getMessage(), e); } return ret; } });
@Override public Long getValue() { long ret = 0L; try { String ufsDataFolder = MasterContext.getConf().get(Constants.UNDERFS_ADDRESS); UnderFileSystem ufs = UnderFileSystem.get(ufsDataFolder, MasterContext.getConf()); ret = ufs.getSpace(ufsDataFolder, UnderFileSystem.SpaceType.SPACE_FREE); } catch (IOException e) { LOG.error(e.getMessage(), e); } return ret; } });
private void connectToUFS() throws IOException { TachyonConf conf = MasterContext.getConf(); String ufsAddress = conf.get(Constants.UNDERFS_ADDRESS); UnderFileSystem ufs = UnderFileSystem.get(ufsAddress, conf); ufs.connectFromMaster(conf, NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, conf)); } }
/** * Starts both a master and a worker using the configurations in {@link MasterContext} and * {@link WorkerContext} respectively. * * @throws IOException when the operation fails */ public void start() throws IOException { mTachyonHome = File.createTempFile("Tachyon", "U" + System.currentTimeMillis()).getAbsolutePath(); // Delete the temp dir by ufs, otherwise, permission problem may be encountered. UnderFileSystemUtils.deleteDir(mTachyonHome, MasterContext.getConf()); mWorkerDataFolder = "/datastore"; mLocalhostName = NetworkAddressUtils.getLocalHostName(100); // Disable hdfs client caching to avoid file system close() affecting other clients System.setProperty("fs.hdfs.impl.disable.cache", "true"); startMaster(); UnderFileSystemUtils.mkdirIfNotExists(mMasterConf.get(Constants.UNDERFS_ADDRESS), mMasterConf); CommonUtils.sleepMs(10); startWorker(); }