public static void setReadColumns(Configuration configuration, List<Integer> readHiveColumnIndexes) { configuration.set(READ_COLUMN_IDS_CONF_STR, Joiner.on(',').join(readHiveColumnIndexes)); configuration.setBoolean(READ_ALL_COLUMNS, false); }
Algorithm(String name) { this.conf = new Configuration(); this.conf.setBoolean("io.native.lib.available", true); this.compressName = name; }
public void updateConfiguration(Configuration config) { copy(resourcesConfiguration, config); // this is to prevent dfs client from doing reverse DNS lookups to determine whether nodes are rack local config.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, NoOpDNSToSwitchMapping.class, DNSToSwitchMapping.class); if (socksProxy != null) { config.setClass(HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY, SocksSocketFactory.class, SocketFactory.class); config.set(HADOOP_SOCKS_SERVER_KEY, socksProxy.toString()); } if (domainSocketPath != null) { config.setStrings(DFS_DOMAIN_SOCKET_PATH_KEY, domainSocketPath); } // only enable short circuit reads if domain socket path is properly configured if (!config.get(DFS_DOMAIN_SOCKET_PATH_KEY, "").trim().isEmpty()) { config.setBooleanIfUnset(DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); } config.setInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, toIntExact(dfsTimeout.toMillis())); config.setInt(IPC_PING_INTERVAL_KEY, toIntExact(ipcPingInterval.toMillis())); config.setInt(IPC_CLIENT_CONNECT_TIMEOUT_KEY, toIntExact(dfsConnectTimeout.toMillis())); config.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, dfsConnectMaxRetries); if (isHdfsWireEncryptionEnabled) { config.set(HADOOP_RPC_PROTECTION, "privacy"); config.setBoolean("dfs.encrypt.data.transfer", true); } config.setInt("fs.cache.max-size", fileSystemMaxCacheSize); config.setInt(LineRecordReader.MAX_LINE_LENGTH, textMaxLineLength); configureCompression(config, compressionCodec); s3ConfigurationUpdater.updateConfiguration(config); }
@Test public void testWithBZip2CodeWithHadoopConfig() throws IOException { Configuration conf = new Configuration(); conf.setBoolean("mapred.output.compress", true); conf.set("mapred.output.compression.codec","org.apache.hadoop.io.compress.BZip2Codec"); conf.setInt(SYNC_INTERVAL_KEY, TEST_SYNC_INTERVAL); testGetRecordWriter(conf, CodecFactory.bzip2Codec(), TEST_SYNC_INTERVAL); }
@Test public void testNoRegionOnMaster() throws Exception { c.setBoolean(LoadBalancer.TABLES_ON_MASTER, false); c.setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, false); int rsCount = (REGIONS + SYSTEM_REGIONS)/SLAVES; checkBalance(0, rsCount); }
@BeforeClass public static void beforeClass() throws Exception { UTIL.getConfiguration().setBoolean(COALESCE_INC_KEY, true); UTIL.getConfiguration().setBoolean("hbase.table.sanity.checks", false); UTIL.getConfiguration().setInt("hbase.client.retries.number", 3); UTIL.startMiniCluster(); }
@Test public void testNonExplicitRCFileHeader() throws IOException, SerDeException { Configuration conf = new Configuration(); conf.setBoolean(HiveConf.ConfVars.HIVEUSEEXPLICITRCFILEHEADER.varname, false); char[] expected = new char[] {'S', 'E', 'Q'}; testRCFileHeader(expected, conf); }
@Test public void testExportFileSystemStateWithSkipTmp() throws Exception { TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true); try { testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); } finally { TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false); } }
@Test public void testRpcFallbackToSimpleAuth() throws Exception { String clientUsername = "testuser"; UserGroupInformation clientUgi = UserGroupInformation.createUserForTesting(clientUsername, new String[] { clientUsername }); // check that the client user is insecure assertNotSame(ugi, clientUgi); assertEquals(AuthenticationMethod.SIMPLE, clientUgi.getAuthenticationMethod()); assertEquals(clientUsername, clientUgi.getUserName()); clientConf.set(User.HBASE_SECURITY_CONF_KEY, "simple"); serverConf.setBoolean(RpcServer.FALLBACK_TO_INSECURE_CLIENT_AUTH, true); callRpcService(User.create(clientUgi)); }
/** * Create configuration for test. * * @param authority Authority. * @param skipEmbed Whether to skip embedded mode. * @param skipLocShmem Whether to skip local shmem mode. * @return Configuration. */ protected Configuration configurationSecondary(String authority) { Configuration cfg = new Configuration(); cfg.set("fs.defaultFS", "igfs://" + authority + "/"); cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName()); cfg.set("fs.AbstractFileSystem.igfs.impl", org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName()); cfg.setBoolean("fs.igfs.impl.disable.cache", true); cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true); cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true); return cfg; } }
@BeforeClass public static void setupBeforeClass() throws Exception { conf = TEST_UTIL.getConfiguration(); // Enable favored nodes based load balancer conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, LoadOnlyFavoredStochasticBalancer.class, LoadBalancer.class); conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 30000); conf.setInt("hbase.master.balancer.stochastic.moveCost", 0); conf.setBoolean("hbase.master.balancer.stochastic.execute.maxSteps", true); conf.set(BaseLoadBalancer.TABLES_ON_MASTER, "none"); }
private void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); conf.set("hbase.balancer.tablesOnMaster", "none"); conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3); conf.setInt(HConstants.HBASE_CLIENT_SERVERSIDE_RETRIES_MULTIPLIER, 3); conf.setBoolean("hbase.split.writer.creation.bounded", true); conf.setInt("hbase.regionserver.hlog.splitlog.writer.threads", 8); LOG.info("WAL splitting coordinated by zk? {}", splitWALCoordinatedByZK); conf.setBoolean(HConstants.HBASE_SPLIT_WAL_COORDINATED_BY_ZK, splitWALCoordinatedByZK); }
@BeforeClass public static void before() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName()); conf.setStrings( CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName(), CPMasterObserver.class.getName()); conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5); conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class, RegionServerObserver.class); UTIL.startMiniCluster(); waitForQuotaInitialize(UTIL); ADMIN = UTIL.getAdmin(); }
/** {@inheritDoc} */ @Override protected Configuration configuration(String authority, boolean skipEmbed, boolean skipLocShmem) { Configuration cfg = new Configuration(); cfg.set("fs.defaultFS", "igfs://" + authority + "/"); cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName()); cfg.set("fs.AbstractFileSystem.igfs.impl", org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName()); cfg.setBoolean("fs.igfs.impl.disable.cache", true); cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true); cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true); cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_TCP, authority), true); cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_REMOTE_TCP, authority), true); cfg.setStrings(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_IGNITE_CFG_PATH, authority), getClientConfig()); return cfg; }
protected static void initTestingUtility(HBaseTestingUtility util, String zkParent) { util.setZkCluster(ZK_UTIL.getZkCluster()); Configuration conf = util.getConfiguration(); conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkParent); conf.setInt("replication.source.size.capacity", 102400); conf.setLong("replication.source.sleepforretries", 100); conf.setInt("hbase.regionserver.maxlogs", 10); conf.setLong("hbase.master.logcleaner.ttl", 10); conf.setInt("zookeeper.recovery.retry", 1); conf.setInt("zookeeper.recovery.retry.intervalmill", 10); conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); conf.setInt("replication.stats.thread.period.seconds", 5); conf.setBoolean("hbase.tests.use.shortcircuit.reads", false); conf.setLong("replication.sleep.before.failover", 2000); conf.setInt("replication.source.maxretriesmultiplier", 10); conf.setFloat("replication.source.ratio", 1.0f); conf.setBoolean("replication.source.eof.autorecovery", true); }
public static void setUpTestBeforeClass() throws Exception { TEST_UTIL = new HBaseTestingUtility(); TEST_UTIL.getConfiguration().setFloat( "hbase.master.balancer.stochastic.tableSkewCost", 6000); TEST_UTIL.getConfiguration().set( HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName()); TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, RSGroupAdminEndpoint.class.getName() + "," + CPMasterObserver.class.getName()); TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1); TEST_UTIL.getConfiguration().setInt( ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_SLAVES_BASE - 1); TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); initialize(); }
@Before public void setUp() throws IOException { conf = TEST_UTIL.getConfiguration(); this.conf.set("dfs.datanode.data.dir.perm", "700"); conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, INDEX_BLOCK_SIZE); conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, BLOOM_BLOCK_SIZE); conf.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, cacheCompressedData); cowType.modifyConf(conf); conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, cowType.shouldBeCached(BlockType.DATA)); conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, cowType.shouldBeCached(BlockType.LEAF_INDEX)); conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, cowType.shouldBeCached(BlockType.BLOOM_CHUNK)); cacheConf = new CacheConfig(conf, blockCache); fs = HFileSystem.get(conf); }
protected static void setupConfiguration() { TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10); TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); }
@Test public void testWithSnappyCode() throws IOException { Configuration conf = new Configuration(); conf.setBoolean("mapred.output.compress", true); conf.set(AvroJob.CONF_OUTPUT_CODEC, DataFileConstants.SNAPPY_CODEC); conf.setInt(SYNC_INTERVAL_KEY, TEST_SYNC_INTERVAL); testGetRecordWriter(conf, CodecFactory.snappyCodec(), TEST_SYNC_INTERVAL); }
@Test public void testRegionsOnAllServers() throws Exception { c.setBoolean(LoadBalancer.TABLES_ON_MASTER, true); c.setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, false); int rsCount = (REGIONS + SYSTEM_REGIONS)/(SLAVES + 1/*Master*/); checkBalance(rsCount, rsCount); }