protected HiveRegister(State state) { this.props = new HiveRegProps(state); this.hiveDbRootDir = this.props.getDbRootDir(); this.executor = ExecutorsUtils.loggingDecorator( ScalingThreadPoolExecutor.newScalingThreadPool(0, this.props.getNumThreads(), TimeUnit.SECONDS.toMillis(10), ExecutorsUtils.newThreadFactory(Optional.of(log), Optional.of(getClass().getSimpleName())))); }
/** * @param state This is a Job State */ public HiveRegistrationPublisher(State state) { super(state); this.hiveRegister = this.closer.register(HiveRegister.get(state)); this.hivePolicyExecutor = ExecutorsUtils.loggingDecorator(Executors.newFixedThreadPool(new HiveRegProps(state).getNumThreads(), ExecutorsUtils.newThreadFactory(Optional.of(log), Optional.of("HivePolicyExecutor-%d")))); this.metricContext = Instrumented.getMetricContext(state, HiveRegistrationPublisher.class); isPathDedupeEnabled = state.getPropAsBoolean(PATH_DEDUPE_ENABLED, this.DEFAULT_PATH_DEDUPE_ENABLED); }
public HiveRegistrationPolicyBase(State props) throws IOException { Preconditions.checkNotNull(props); this.props = new HiveRegProps(props); if (props.contains(HiveRegistrationPolicyBase.HIVE_FS_URI)) { this.fs = FileSystem.get(URI.create(props.getProp(HiveRegistrationPolicyBase.HIVE_FS_URI)), new Configuration()); } else { this.fs = FileSystem.get(new Configuration()); } this.sanitizeNameAllowed = props.getPropAsBoolean(HIVE_SANITIZE_INVALID_NAMES, true); this.dbNamePattern = props.contains(HIVE_DATABASE_REGEX) ? Optional.of(Pattern.compile(props.getProp(HIVE_DATABASE_REGEX))) : Optional.<Pattern> absent(); this.tableNamePattern = props.contains(HIVE_TABLE_REGEX) ? Optional.of(Pattern.compile(props.getProp(HIVE_TABLE_REGEX))) : Optional.<Pattern> absent(); this.dbNamePrefix = props.getProp(HIVE_DATABASE_NAME_PREFIX, StringUtils.EMPTY); this.dbNameSuffix = props.getProp(HIVE_DATABASE_NAME_SUFFIX, StringUtils.EMPTY); this.tableNamePrefix = props.getProp(HIVE_TABLE_NAME_PREFIX, StringUtils.EMPTY); this.tableNameSuffix = props.getProp(HIVE_TABLE_NAME_SUFFIX, StringUtils.EMPTY); this.emptyInputPathFlag = props.getPropAsBoolean(MAPREDUCE_JOB_INPUT_PATH_EMPTY_KEY, false); this.metricContext = Instrumented.getMetricContext(props, HiveRegister.class); }
@Test public void testAddTableDeregisterSteps() throws Exception { HiveDataset dataset = Mockito.mock(HiveDataset.class); Mockito.when(dataset.getProperties()).thenReturn(new Properties()); HiveCopyEntityHelper helper = Mockito.mock(HiveCopyEntityHelper.class); Mockito.when(helper.getDeleteMethod()).thenReturn(DeregisterFileDeleteMethod.NO_DELETE); Mockito.when(helper.getTargetURI()).thenReturn(Optional.of("/targetURI")); Mockito.when(helper.getHiveRegProps()).thenReturn(new HiveRegProps(new State())); Mockito.when(helper.getDataset()).thenReturn(dataset); Mockito.when(helper.addTableDeregisterSteps(Mockito.any(List.class), Mockito.any(String.class), Mockito.anyInt(), Mockito.any(org.apache.hadoop.hive.ql.metadata.Table.class))).thenCallRealMethod(); org.apache.hadoop.hive.ql.metadata.Table meta_table = Mockito.mock(org.apache.hadoop.hive.ql.metadata.Table.class); org.apache.hadoop.hive.metastore.api.Table api_table = Mockito.mock(org.apache.hadoop.hive.metastore.api.Table.class); Mockito.when(api_table.getDbName()).thenReturn("TestDB"); Mockito.when(api_table.getTableName()).thenReturn("TestTable"); Mockito.when(meta_table.getTTable()).thenReturn(api_table); List<CopyEntity> copyEntities = new ArrayList<CopyEntity>(); String fileSet = "testFileSet"; int initialPriority = 0; int priority = helper.addTableDeregisterSteps(copyEntities, fileSet, initialPriority, meta_table); Assert.assertTrue(priority == 1); Assert.assertTrue(copyEntities.size() == 1); Assert.assertTrue(copyEntities.get(0) instanceof PostPublishStep); PostPublishStep p = (PostPublishStep) (copyEntities.get(0)); Assert .assertTrue(p.getStep().toString().contains("Deregister table TestDB.TestTable on Hive metastore /targetURI")); }
this.hiveRegProps = new HiveRegProps(new State(this.dataset.getProperties())); this.targetURI = Optional.fromNullable(this.dataset.getProperties().getProperty(TARGET_METASTORE_URI_KEY)); this.targetClientPool = HiveMetastoreClientPool.get(this.dataset.getProperties(), this.targetURI);
/** * Constructor for {@link HiveMetastoreClientPool}. * @deprecated It is recommended to use the static {@link #get} method instead. Use this constructor only if you * different pool configurations are required. */ @Deprecated public HiveMetastoreClientPool(Properties properties, Optional<String> metastoreURI) { this.hiveRegProps = new HiveRegProps(new State(properties)); GenericObjectPoolConfig config = new GenericObjectPoolConfig(); config.setMaxTotal(this.hiveRegProps.getNumThreads()); config.setMaxIdle(this.hiveRegProps.getNumThreads()); this.factory = new HiveMetaStoreClientFactory(metastoreURI); this.pool = new GenericObjectPool<>(this.factory, config); this.hiveConf = this.factory.getHiveConf(); }
protected HiveRegister(State state) { this.props = new HiveRegProps(state); this.hiveDbRootDir = this.props.getDbRootDir(); this.executor = ExecutorsUtils.loggingDecorator( ScalingThreadPoolExecutor.newScalingThreadPool(0, this.props.getNumThreads(), TimeUnit.SECONDS.toMillis(10), ExecutorsUtils.newThreadFactory(Optional.of(log), Optional.of(getClass().getSimpleName())))); }
/** * @param state This is a Job State */ public HiveRegistrationPublisher(State state) { super(state); this.hiveRegister = this.closer.register(HiveRegister.get(state)); this.hivePolicyExecutor = ExecutorsUtils.loggingDecorator(Executors.newFixedThreadPool(new HiveRegProps(state).getNumThreads(), ExecutorsUtils.newThreadFactory(Optional.of(log), Optional.of("HivePolicyExecutor-%d")))); this.metricContext = Instrumented.getMetricContext(state, HiveRegistrationPublisher.class); isPathDedupeEnabled = state.getPropAsBoolean(PATH_DEDUPE_ENABLED, this.DEFAULT_PATH_DEDUPE_ENABLED); }
public HiveRegistrationPolicyBase(State props) throws IOException { Preconditions.checkNotNull(props); this.props = new HiveRegProps(props); if (props.contains(HiveRegistrationPolicyBase.HIVE_FS_URI)) { this.fs = FileSystem.get(URI.create(props.getProp(HiveRegistrationPolicyBase.HIVE_FS_URI)), new Configuration()); } else { this.fs = FileSystem.get(new Configuration()); } this.sanitizeNameAllowed = props.getPropAsBoolean(HIVE_SANITIZE_INVALID_NAMES, true); this.dbNamePattern = props.contains(HIVE_DATABASE_REGEX) ? Optional.of(Pattern.compile(props.getProp(HIVE_DATABASE_REGEX))) : Optional.<Pattern> absent(); this.tableNamePattern = props.contains(HIVE_TABLE_REGEX) ? Optional.of(Pattern.compile(props.getProp(HIVE_TABLE_REGEX))) : Optional.<Pattern> absent(); this.dbNamePrefix = props.getProp(HIVE_DATABASE_NAME_PREFIX, StringUtils.EMPTY); this.dbNameSuffix = props.getProp(HIVE_DATABASE_NAME_SUFFIX, StringUtils.EMPTY); this.tableNamePrefix = props.getProp(HIVE_TABLE_NAME_PREFIX, StringUtils.EMPTY); this.tableNameSuffix = props.getProp(HIVE_TABLE_NAME_SUFFIX, StringUtils.EMPTY); this.emptyInputPathFlag = props.getPropAsBoolean(MAPREDUCE_JOB_INPUT_PATH_EMPTY_KEY, false); this.metricContext = Instrumented.getMetricContext(props, HiveRegister.class); }
this.hiveRegProps = new HiveRegProps(new State(this.dataset.getProperties())); this.targetURI = Optional.fromNullable(this.dataset.getProperties().getProperty(TARGET_METASTORE_URI_KEY)); this.targetClientPool = HiveMetastoreClientPool.get(this.dataset.getProperties(), this.targetURI);
/** * Constructor for {@link HiveMetastoreClientPool}. * @deprecated It is recommended to use the static {@link #get} method instead. Use this constructor only if you * different pool configurations are required. */ @Deprecated public HiveMetastoreClientPool(Properties properties, Optional<String> metastoreURI) { this.hiveRegProps = new HiveRegProps(new State(properties)); GenericObjectPoolConfig config = new GenericObjectPoolConfig(); config.setMaxTotal(this.hiveRegProps.getNumThreads()); config.setMaxIdle(this.hiveRegProps.getNumThreads()); this.factory = new HiveMetaStoreClientFactory(metastoreURI); this.pool = new GenericObjectPool<>(this.factory, config); this.hiveConf = this.factory.getHiveConf(); }