public HelixInstanceDataManagerConfig(Configuration serverConfig) throws ConfigurationException { _instanceDataManagerConfiguration = serverConfig; Iterator keysIterator = serverConfig.getKeys(); while (keysIterator.hasNext()) { String key = (String) keysIterator.next(); LOGGER.info("InstanceDataManagerConfig, key: {} , value: {}", key, serverConfig.getProperty(key)); } checkRequiredKeys(); }
public static ServerConf getDefaultHelixServerConfig(Configuration externalConfigs) { Configuration defaultConfigs = loadDefaultServerConf(); @SuppressWarnings("unchecked") Iterator<String> iterable = externalConfigs.getKeys(); while (iterable.hasNext()) { String key = iterable.next(); defaultConfigs.setProperty(key, externalConfigs.getProperty(key)); LOGGER.info("External config key: {}, value: {}", key, externalConfigs.getProperty(key)); } return new ServerConf(defaultConfigs); }
private static void copyInputKeys(org.apache.hadoop.conf.Configuration hadoopConf, org.apache.commons.configuration.Configuration source) { // Copy IndexUpdateJob settings into the hadoop-backed cfg Iterator<String> keyIter = source.getKeys(); while (keyIter.hasNext()) { String key = keyIter.next(); ConfigElement.PathIdentifier pid; try { pid = ConfigElement.parse(ROOT_NS, key); } catch (RuntimeException e) { log.debug("[inputkeys] Skipping {}", key, e); continue; } if (!pid.element.isOption()) continue; String k = ConfigElement.getPath(TitanHadoopConfiguration.GRAPH_CONFIG_KEYS, true) + "." + key; String v = source.getProperty(key).toString(); hadoopConf.set(k, v); log.debug("[inputkeys] Set {}={}", k, v); } }
public static Configuration getDefaultBrokerConf(Configuration externalConfigs) { final Configuration defaultConfigs = getDefaultBrokerConf(); @SuppressWarnings("unchecked") Iterator<String> iterable = externalConfigs.getKeys(); while (iterable.hasNext()) { String key = iterable.next(); defaultConfigs.setProperty(key, externalConfigs.getProperty(key)); } return defaultConfigs; } }
/** * Load configurations with prefixed <i>section</i> from source configuration <i>srcConf</i> into * target configuration <i>targetConf</i>. * * @param targetConf * Target Configuration * @param srcConf * Source Configuration * @param section * Section Key */ public static void loadConfiguration(Configuration targetConf, Configuration srcConf, String section) { Iterator confKeys = srcConf.getKeys(); while (confKeys.hasNext()) { Object keyObject = confKeys.next(); if (!(keyObject instanceof String)) { continue; } String key = (String) keyObject; if (key.startsWith(section)) { targetConf.setProperty(key.substring(section.length()), srcConf.getProperty(key)); } } }
Object o = config.getProperty(key); if (datatype.isInstance(o)) { return (O)o; Preconditions.checkState(null != constants && 0 < constants.length, "Zero-length or undefined enum"); String estr = config.getProperty(key).toString(); for (Enum ec : constants) if (ec.toString().equals(estr)) throw new IllegalArgumentException("No match for string \"" + estr + "\" in enum " + datatype); } else if (datatype==Object.class) { return (O)config.getProperty(key); } else if (Duration.class.isAssignableFrom(datatype)) { Object o = config.getProperty(key); if (Duration.class.isInstance(o)) { return (O) o;
public void setPinotFSFactoryClasses(Configuration pinotFSFactoryClasses) { setProperty(PINOT_FS_FACTORY_CLASS_LOCAL, DEFAULT_PINOT_FS_FACTORY_CLASS_LOCAL); if (pinotFSFactoryClasses != null) { pinotFSFactoryClasses.getKeys() .forEachRemaining(key -> setProperty((String) key, pinotFSFactoryClasses.getProperty((String) key))); } }
public static void init(Configuration fsConfig) { // Get schemes and their respective classes Iterator<String> keys = fsConfig.subset(CLASS).getKeys(); if (!keys.hasNext()) { LOGGER.info("Did not find any fs classes in the configuration"); } while (keys.hasNext()) { String key = keys.next(); String fsClassName = (String) fsConfig.getProperty(CLASS + "." + key); LOGGER.info("Got scheme {}, classname {}, starting to initialize", key, fsClassName); try { PinotFS pinotFS = (PinotFS) Class.forName(fsClassName).newInstance(); pinotFS.init(fsConfig.subset(key)); LOGGER.info("Initializing PinotFS for scheme {}, classname {}", key, fsClassName); _fileSystemMap.put(key, pinotFS); } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { LOGGER.error("Could not instantiate file system for class {}", fsClassName, e); throw new RuntimeException(e); } } if (!_fileSystemMap.containsKey(DEFAULT_FS_SCHEME)) { LOGGER.info("LocalPinotFS not configured, adding as default"); _fileSystemMap.put(DEFAULT_FS_SCHEME, new LocalPinotFS()); } }
String className = (String) config.getProperty(CLASS + "." + key); LOGGER.info("Got crypter class name {}, full crypter path {}, starting to initialize", key, className);
Object o = config.getProperty(key); if (dataType.isInstance(o)) { return (O)o; Preconditions.checkState(null != constants && 0 < constants.length, "Zero-length or undefined enum"); String enumString = config.getProperty(key).toString(); for (Enum ec : constants) if (ec.toString().equals(enumString)) throw new IllegalArgumentException("No match for string \"" + enumString + "\" in enum " + dataType); } else if (dataType==Object.class) { return (O)config.getProperty(key); } else if (Duration.class.isAssignableFrom(dataType)) { Object o = config.getProperty(key); if (Duration.class.isInstance(o)) { return (O) o;
private BrokerServerBuilder startBroker(Configuration config) { if (config == null) { config = DefaultHelixBrokerConfig.getDefaultBrokerConf(); } BrokerServerBuilder brokerServerBuilder = new BrokerServerBuilder(config, _helixExternalViewBasedRouting, _helixExternalViewBasedRouting.getTimeBoundaryService(), _liveInstancesListener, _tableQueryQuotaManager); _accessControlFactory = brokerServerBuilder.getAccessControlFactory(); _helixExternalViewBasedRouting.setBrokerMetrics(brokerServerBuilder.getBrokerMetrics()); _tableQueryQuotaManager.setBrokerMetrics(brokerServerBuilder.getBrokerMetrics()); brokerServerBuilder.start(); LOGGER.info("Pinot broker ready and listening on port {} for API requests", config.getProperty("pinot.broker.client.queryPort")); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { brokerServerBuilder.stop(); } catch (final Exception e) { LOGGER.error("Caught exception while running shutdown hook", e); } } }); return brokerServerBuilder; }
public static Configuration makeHadoopConfiguration(final org.apache.commons.configuration.Configuration apacheConfiguration) { final Configuration hadoopConfiguration = new Configuration(); apacheConfiguration.getKeys().forEachRemaining(key -> { final Object object = apacheConfiguration.getProperty(key); hadoopConfiguration.set(key, object.toString()); }); return hadoopConfiguration; }
public static void mergeApacheIntoHadoopConfiguration(final org.apache.commons.configuration.Configuration apacheConfiguration, final Configuration hadoopConfiguration) { apacheConfiguration.getKeys().forEachRemaining(key -> { final Object object = apacheConfiguration.getProperty(key); hadoopConfiguration.set(key, object.toString()); }); }
public static Map<String, String> getPropertiesWithPrefix(Configuration configuration, String prefix) { Map<String, String> propertiesMap = new HashMap<>(); Iterator<String> keysIterator = configuration.getKeys(prefix); while (keysIterator.hasNext()) { String key = keysIterator.next(); propertiesMap.put(key.substring(prefix.length() + 1), String.valueOf(configuration.getProperty(key))); } return propertiesMap; }
public static SparkContext create(final Configuration configuration) { final SparkConf sparkConf = new SparkConf(); configuration.getKeys().forEachRemaining(key -> sparkConf.set(key, configuration.getProperty(key).toString())); return Spark.create(sparkConf); }
public static <S, E> PureTraversal<S, E> loadState(final Configuration configuration, final String configurationKey, final Graph graph) { final Object configValue = configuration.getProperty(configurationKey); final PureTraversal<S, E> pureTraversal = (configValue instanceof String ? (PureTraversal<S, E>) VertexProgramHelper.deserialize(configuration, configurationKey) : ((PureTraversal<S, E>) configValue)); pureTraversal.pureTraversal.setGraph(graph); return pureTraversal; }
private static Configuration maskedConfiguration(final Configuration configuration) { final Configuration maskedConfiguration = new BaseConfiguration(); final Iterator keys = configuration.getKeys(); while(keys.hasNext()) { final String key = (String)keys.next(); if (key.matches(maskedProperties)) maskedConfiguration.setProperty(key, "******"); else maskedConfiguration.setProperty(key, configuration.getProperty(key)); } return maskedConfiguration; }
public static SubgraphStrategy create(final Configuration configuration) { final Builder builder = SubgraphStrategy.build(); if (configuration.containsKey(VERTICES)) builder.vertices((Traversal) configuration.getProperty(VERTICES)); if (configuration.containsKey(EDGES)) builder.edges((Traversal) configuration.getProperty(EDGES)); if (configuration.containsKey(VERTEX_PROPERTIES)) builder.vertexProperties((Traversal) configuration.getProperty(VERTEX_PROPERTIES)); if (configuration.containsKey(CHECK_ADJACENT_VERTICES)) builder.checkAdjacentVertices(configuration.getBoolean(CHECK_ADJACENT_VERTICES)); return builder.create(); }
public static PartitionStrategy create(final Configuration configuration) { final PartitionStrategy.Builder builder = PartitionStrategy.build(); if (configuration.containsKey(INCLUDE_META_PROPERTIES)) builder.includeMetaProperties(configuration.getBoolean(INCLUDE_META_PROPERTIES)); if (configuration.containsKey(WRITE_PARTITION)) builder.writePartition(configuration.getString(WRITE_PARTITION)); if (configuration.containsKey(PARTITION_KEY)) builder.partitionKey(configuration.getString(PARTITION_KEY)); if (configuration.containsKey(READ_PARTITIONS)) builder.readPartitions(new ArrayList((Collection)configuration.getProperty(READ_PARTITIONS))); return builder.create(); }
public static ElementIdStrategy create(final Configuration configuration) { final ElementIdStrategy.Builder builder = ElementIdStrategy.build(); if (configuration.containsKey(ID_MAKER)) builder.idMaker((Supplier) configuration.getProperty(ID_MAKER)); if (configuration.containsKey(ID_PROPERTY_KEY)) builder.idPropertyKey(configuration.getString(ID_PROPERTY_KEY)); return builder.create(); }