/** * Sets the value for the appropriate key in the {@link Properties}. * * @param key the key to set * @param value the value for the key */ public static void set(PropertyKey key, Object value) { set(key, String.valueOf(value), Source.RUNTIME); }
/** * @param bucketPath the bucket path like "/bucket", "/mount/point/bucket" * @param objectKey the object key like "img/2017/9/1/s3.jpg" * @return the temporary directory used to hold parts of the object during multipart uploads */ public static String getMultipartTemporaryDirForObject(String bucketPath, String objectKey) { String multipartTemporaryDirSuffix = Configuration.get(PropertyKey.PROXY_S3_MULTIPART_TEMPORARY_DIR_SUFFIX); return bucketPath + AlluxioURI.SEPARATOR + objectKey + multipartTemporaryDirSuffix; }
/** * Checks if authorization is enabled. * * @return true if authorization is enabled, false otherwise */ public static boolean isAuthorizationEnabled() { return Configuration.getBoolean(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED); }
/** * @return whether the configuration describes how to find the master host, either through * explicit configuration or through zookeeper */ public static boolean masterHostConfigured() { boolean usingZk = Configuration.getBoolean(PropertyKey.ZOOKEEPER_ENABLED) && Configuration.isSet(PropertyKey.ZOOKEEPER_ADDRESS); return Configuration.isSet(PropertyKey.MASTER_HOSTNAME) || usingZk; }
/** * Constructs a {@link PermissionChecker} instance for Alluxio file system. * * @param inodeTree inode tree of the file system master */ public DefaultPermissionChecker(InodeTree inodeTree) { mInodeTree = Preconditions.checkNotNull(inodeTree, "inodeTree"); mPermissionCheckEnabled = Configuration.getBoolean(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED); mFileSystemSuperGroup = Configuration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_SUPERGROUP); mInodePermissionChecker = InodePermissionChecker.create(); }
/** * @return the instance of under file system for Alluxio root directory */ public static UnderFileSystem createForRoot() { String ufsRoot = Configuration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); boolean readOnly = Configuration.getBoolean(PropertyKey.MASTER_MOUNT_TABLE_ROOT_READONLY); boolean shared = Configuration.getBoolean(PropertyKey.MASTER_MOUNT_TABLE_ROOT_SHARED); Map<String, String> ufsConf = Configuration.getNestedProperties(PropertyKey.MASTER_MOUNT_TABLE_ROOT_OPTION); return create(ufsRoot, UnderFileSystemConfiguration.defaults().setReadOnly(readOnly) .setShared(shared).setUserSpecifiedConf(ufsConf)); } }
mMinWorkerThreads = Configuration.getInt(PropertyKey.MASTER_WORKER_THREADS_MIN); mMaxWorkerThreads = Configuration.getInt(PropertyKey.MASTER_WORKER_THREADS_MAX); int connectionTimeout = (int) Configuration.getMs(PropertyKey.MASTER_CONNECTION_TIMEOUT_MS); if (!Configuration.getBoolean(PropertyKey.TEST_MODE)) { Preconditions.checkState(Configuration.getInt(PropertyKey.MASTER_RPC_PORT) > 0, this + " rpc port is only allowed to be zero in test mode."); Preconditions.checkState(Configuration.getInt(PropertyKey.MASTER_WEB_PORT) > 0, this + " web port is only allowed to be zero in test mode."); mPort = ThriftUtils.getThriftPort(mRpcServerSocket); Configuration.set(PropertyKey.MASTER_RPC_PORT, Integer.toString(mPort)); mRpcBindAddress = NetworkAddressUtils.getBindAddress(ServiceType.MASTER_RPC); mRpcConnectAddress = NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC);
HadoopConfigurationUtils.mergeHadoopConfiguration(conf, Configuration.global()); Configuration.set(PropertyKey.ZOOKEEPER_ENABLED, isZookeeperMode()); if (!Configuration.getBoolean(PropertyKey.ZOOKEEPER_ENABLED)) { Preconditions.checkNotNull(uri.getHost(), PreconditionMessage.URI_HOST_NULL); Preconditions.checkNotNull(uri.getPort(), PreconditionMessage.URI_PORT_NULL); Configuration.set(PropertyKey.MASTER_HOSTNAME, uri.getHost()); Configuration.set(PropertyKey.MASTER_RPC_PORT, uri.getPort()); FileSystemContext.get().reset(Configuration.global());
/** * Creates a new instance with defaults from the configuration. */ private CreateFileOptions() { Configuration conf = ClientContext.getConf(); mRecursive = true; mBlockSizeBytes = conf.getBytes(Constants.USER_BLOCK_SIZE_BYTES_DEFAULT); try { mLocationPolicy = CommonUtils.createNewClassInstance( conf.<FileWriteLocationPolicy>getClass(Constants.USER_FILE_WRITE_LOCATION_POLICY), new Class[]{}, new Object[]{}); } catch (Exception e) { throw Throwables.propagate(e); } mWriteType = conf.getEnum(Constants.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.class); mTtl = Constants.NO_TTL; }
/** * Block size for reading an object in chunks. * * @return block size in bytes */ private long getBlockSize() { return Configuration.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT); }
/** * Constructs an {@link AlluxioLogServerProcess} instance. * * @param baseLogsDir base directory to store the logs pushed from remote Alluxio servers */ public AlluxioLogServerProcess(String baseLogsDir) { mPort = Configuration.getInt(PropertyKey.LOGSERVER_PORT); // The log server serves the logging requests from Alluxio servers. mMinNumberOfThreads = Configuration.getInt(PropertyKey.LOGSERVER_THREADS_MIN); mMaxNumberOfThreads = Configuration.getInt(PropertyKey.LOGSERVER_THREADS_MAX); mBaseLogsDir = baseLogsDir; mStopped = true; }
/** * Initializes the client context singleton, bringing all non-Alluxio configuration state in sync * with the current Alluxio configuration. * * This method is useful for updating parts of {@link ClientContext} which depend on * {@link Configuration} when {@link Configuration} is changed, e.g. the master hostname or port. * This method requires that configuration has been initialized. */ public static void init() { String masterHostname = Preconditions.checkNotNull(sConf.get(Constants.MASTER_HOSTNAME)); int masterPort = sConf.getInt(Constants.MASTER_RPC_PORT); sMasterAddress = new InetSocketAddress(masterHostname, masterPort); sClientMetrics = new ClientMetrics(); sExecutorService = Executors.newFixedThreadPool( sConf.getInt(Constants.USER_BLOCK_WORKER_CLIENT_THREADS), ThreadFactoryUtils.build("block-worker-heartbeat-%d", true)); }
@Override protected void startMasters() throws IOException { Configuration.set(PropertyKey.ZOOKEEPER_ENABLED, "true"); Configuration.set(PropertyKey.ZOOKEEPER_ADDRESS, mCuratorServer.getConnectString()); Configuration.set(PropertyKey.ZOOKEEPER_ELECTION_PATH, "/election"); Configuration.set(PropertyKey.ZOOKEEPER_LEADER_PATH, "/leader"); mMasters.add(master); Configuration.set(PropertyKey.MASTER_RPC_PORT, "0"); String path = Configuration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); if (ufs.isDirectory(path)) { ufs.deleteDirectory(path, DeleteOptions.defaults().setRecursive(true)); Configuration.set(PropertyKey.MASTER_RPC_PORT, String.valueOf(getLocalAlluxioMaster().getRpcLocalPort()));
private WriteType getS3WriteType() { return Configuration.getEnum(PropertyKey.PROXY_S3_WRITE_TYPE, WriteType.class); }
/** * Constructs JvmPauseMonitor. */ public JvmPauseMonitor() { mGcSleepIntervalMs = Configuration.getMs(PropertyKey.JVM_MONITOR_SLEEP_INTERVAL_MS); mWarnThresholdMs = Configuration.getMs(PropertyKey.JVM_MONITOR_WARN_THRESHOLD_MS); mInfoThresholdMs = Configuration.getMs(PropertyKey.JVM_MONITOR_INFO_THRESHOLD_MS); }
private InStreamOptions() { mReadType = ClientContext.getConf().getEnum(Constants.USER_FILE_READ_TYPE_DEFAULT, ReadType.class); try { mLocationPolicy = CommonUtils.createNewClassInstance(ClientContext.getConf() .<FileWriteLocationPolicy>getClass(Constants.USER_FILE_WRITE_LOCATION_POLICY), new Class[]{}, new Object[]{}); } catch (Exception e) { throw Throwables.propagate(e); } }
@Override public InputStream open(String path, OpenOptions options) throws IOException { IOException thrownException = null; RetryPolicy retryPolicy = new ExponentialBackoffRetry( (int) Configuration.getMs(PropertyKey.UNDERFS_OBJECT_STORE_READ_RETRY_BASE_SLEEP_MS), (int) Configuration.getMs(PropertyKey.UNDERFS_OBJECT_STORE_READ_RETRY_MAX_SLEEP_MS), Configuration.getInt(PropertyKey.UNDERFS_OBJECT_STORE_READ_RETRY_MAX_NUM)); while (retryPolicy.attempt()) { try { return openObject(stripPrefixIfPresent(path), options); } catch (IOException e) { LOG.warn("{} attempt to open {} failed with exception : {}", retryPolicy.getAttemptCount(), path, e.getMessage()); thrownException = e; } } throw thrownException; }