/** * Get Hive database root dir from {@link #HIVE_DB_ROOT_DIR}. * * @return {@link Optional#absent()} if {@link #HIVE_DB_ROOT_DIR} is not specified. */ public Optional<String> getDbRootDir() { return Optional.fromNullable(getProp(HIVE_DB_ROOT_DIR)); }
/** * Obtain Hive database names. The returned {@link Iterable} contains the database name returned by * {@link #getDatabaseName(Path)} (if present) plus additional database names specified in * {@link #ADDITIONAL_HIVE_DATABASE_NAMES}. * */ protected Iterable<String> getDatabaseNames(Path path) { List<String> databaseNames = Lists.newArrayList(); Optional<String> databaseName; if ((databaseName = getDatabaseName(path)).isPresent()) { databaseNames.add(databaseName.get()); } if (!Strings.isNullOrEmpty(this.props.getProp(ADDITIONAL_HIVE_DATABASE_NAMES))) { for (String additionalDbName : this.props.getPropAsList(ADDITIONAL_HIVE_DATABASE_NAMES)) { databaseNames.add(this.dbNamePrefix + additionalDbName + this.dbNameSuffix); } } Preconditions.checkState(!databaseNames.isEmpty(), "Hive database name not specified"); return databaseNames; }
tableNames.add(this.tableNamePrefix + resolvedTableName + this.tableNameSuffix); } else if (!Strings.isNullOrEmpty(this.props.getProp(additionalNamesProp))) { for (String additionalTableName : this.props.getPropAsList(additionalNamesProp)) { String resolvedTableName =
tableNames.add(this.tableNamePrefix + resolvedTableName + this.tableNameSuffix); } else if (!Strings.isNullOrEmpty(this.props.getProp(additionalNamesProp))) { for (String additionalTableName : this.props.getPropAsList(additionalNamesProp)) { String resolvedTableName =
/** * Get the name of registered HiveTable's upstream data attributes. * E.g., When data consumed from Kafka is registered into Hive Table, it is expected * to have Hive Metadata indicating the Kafka topic. * * HIVE_UPSTREAM_DATA_ATTR_NAMES is comma separated string, each item representing a upstream data attr. * E.g. hive.upstream.data.attr.names=topic.name,some.else * * @return {@link Optional#absent()} if {@link #HIVE_UPSTREAM_DATA_ATTR_NAMES} is not specified. */ public Optional<String> getUpstreamDataAttrName(){ return Optional.fromNullable(getProp(HIVE_UPSTREAM_DATA_ATTR_NAMES)); }
/** * Obtain Hive database names. The returned {@link Iterable} contains the database name returned by * {@link #getDatabaseName(Path)} (if present) plus additional database names specified in * {@link #ADDITIONAL_HIVE_DATABASE_NAMES}. * */ protected Iterable<String> getDatabaseNames(Path path) { List<String> databaseNames = Lists.newArrayList(); Optional<String> databaseName; if ((databaseName = getDatabaseName(path)).isPresent()) { databaseNames.add(databaseName.get()); } if (!Strings.isNullOrEmpty(this.props.getProp(ADDITIONAL_HIVE_DATABASE_NAMES))) { for (String additionalDbName : this.props.getPropAsList(ADDITIONAL_HIVE_DATABASE_NAMES)) { databaseNames.add(this.dbNamePrefix + additionalDbName + this.dbNameSuffix); } } Preconditions.checkState(!databaseNames.isEmpty(), "Hive database name not specified"); return databaseNames; }
protected HiveRegistrationUnitComparator<?> getTableComparator(HiveTable existingTable, HiveTable newTable) { try { Class<?> clazz = Class.forName(this.props.getProp(HIVE_TABLE_COMPARATOR_TYPE, DEFAULT_HIVE_TABLE_COMPARATOR_TYPE)); return (HiveRegistrationUnitComparator<?>) ConstructorUtils.invokeConstructor(clazz, existingTable, newTable); } catch (ReflectiveOperationException e) { log.error("Unable to instantiate Hive table comparator", e); throw Throwables.propagate(e); } }
protected HiveRegistrationUnitComparator<?> getPartitionComparator(HivePartition existingPartition, HivePartition newPartition) { try { Class<?> clazz = Class.forName(this.props.getProp(HIVE_PARTITION_COMPARATOR_TYPE, DEFAULT_HIVE_PARTITION_COMPARATOR_TYPE)); return (HiveRegistrationUnitComparator<?>) ConstructorUtils.invokeConstructor(clazz, existingPartition, newPartition); } catch (ReflectiveOperationException e) { log.error("Unable to instantiate Hive partition comparator", e); throw Throwables.propagate(e); } }
protected String getDatabaseOrTableName(Path path, String nameKey, String regexKey, Optional<Pattern> pattern) { String name; if (this.props.contains(nameKey)) { name = this.props.getProp(nameKey); } else if (pattern.isPresent()) { Matcher matcher = pattern.get().matcher(path.toString()); if (matcher.matches() && matcher.groupCount() >= 1) { name = matcher.group(1); } else { throw new IllegalStateException("No group match found for regexKey " + regexKey+" with regexp "+ pattern.get().toString() +" on path "+path); } } else { throw new IllegalStateException("Missing required property " + nameKey + " or " + regexKey); } return sanitizeAndValidateName(name); }
/** * Get Hive database root dir from {@link #HIVE_DB_ROOT_DIR}. * * @return {@link Optional#absent()} if {@link #HIVE_DB_ROOT_DIR} is not specified. */ public Optional<String> getDbRootDir() { return Optional.fromNullable(getProp(HIVE_DB_ROOT_DIR)); }
/** * Get the name of registered HiveTable's upstream data attributes. * E.g., When data consumed from Kafka is registered into Hive Table, it is expected * to have Hive Metadata indicating the Kafka topic. * * HIVE_UPSTREAM_DATA_ATTR_NAMES is comma separated string, each item representing a upstream data attr. * E.g. hive.upstream.data.attr.names=topic.name,some.else * * @return {@link Optional#absent()} if {@link #HIVE_UPSTREAM_DATA_ATTR_NAMES} is not specified. */ public Optional<String> getUpstreamDataAttrName(){ return Optional.fromNullable(getProp(HIVE_UPSTREAM_DATA_ATTR_NAMES)); }
protected HiveRegistrationUnitComparator<?> getPartitionComparator(HivePartition existingPartition, HivePartition newPartition) { try { Class<?> clazz = Class.forName(this.props.getProp(HIVE_PARTITION_COMPARATOR_TYPE, DEFAULT_HIVE_PARTITION_COMPARATOR_TYPE)); return (HiveRegistrationUnitComparator<?>) ConstructorUtils.invokeConstructor(clazz, existingPartition, newPartition); } catch (ReflectiveOperationException e) { log.error("Unable to instantiate Hive partition comparator", e); throw Throwables.propagate(e); } }
protected HiveRegistrationUnitComparator<?> getTableComparator(HiveTable existingTable, HiveTable newTable) { try { Class<?> clazz = Class.forName(this.props.getProp(HIVE_TABLE_COMPARATOR_TYPE, DEFAULT_HIVE_TABLE_COMPARATOR_TYPE)); return (HiveRegistrationUnitComparator<?>) ConstructorUtils.invokeConstructor(clazz, existingTable, newTable); } catch (ReflectiveOperationException e) { log.error("Unable to instantiate Hive table comparator", e); throw Throwables.propagate(e); } }
protected String getDatabaseOrTableName(Path path, String nameKey, String regexKey, Optional<Pattern> pattern) { String name; if (this.props.contains(nameKey)) { name = this.props.getProp(nameKey); } else if (pattern.isPresent()) { Matcher matcher = pattern.get().matcher(path.toString()); if (matcher.matches() && matcher.groupCount() >= 1) { name = matcher.group(1); } else { throw new IllegalStateException("No group match found for regexKey " + regexKey+" with regexp "+ pattern.get().toString() +" on path "+path); } } else { throw new IllegalStateException("Missing required property " + nameKey + " or " + regexKey); } return sanitizeAndValidateName(name); }