public static MetricFilter parse(Configs configs, Config config) { Preconditions.checkNotNull(configs); Preconditions.checkNotNull(config); Config filterConfig = configs.getConfig(config, "metricFilter", null); if (filterConfig == null) { return MetricFilter.ALL; } else { return new PatternMetricFilter(filterConfig); } }
public static MetricFilter parse(Configs configs, Config config) { Preconditions.checkNotNull(configs); Preconditions.checkNotNull(config); Config filterConfig = configs.getConfig(config, "metricFilter", null); if (filterConfig == null) { return MetricFilter.ALL; } else { return new PatternMetricFilter(filterConfig); } }
public SanitizeUnknownSolrFields(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); this.schema = locator.getIndexSchema(); Preconditions.checkNotNull(schema); LOG.trace("Solr schema: \n{}", Joiner.on("\n").join(new TreeMap(schema.getFields()).values())); String str = getConfigs().getString(config, "renameToPrefix", "").trim(); this.renameToPrefix = str.length() > 0 ? str : null; validateArguments(); }
public SanitizeUnknownSolrFields(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); this.schema = locator.getIndexSchema(); Preconditions.checkNotNull(schema); LOG.trace("Solr schema: \n{}", Joiner.on("\n").join(new TreeMap(schema.getFields()).values())); String str = getConfigs().getString(config, "renameToPrefix", "").trim(); this.renameToPrefix = str.length() > 0 ? str : null; validateArguments(); }
public SanitizeUnknownSolrFields(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); this.schema = Objects.requireNonNull(locator.getIndexSchema()); if (LOG.isTraceEnabled()) { LOG.trace("Solr schema: \n" + schema.getFields().entrySet().stream().sorted(Map.Entry.comparingByKey()) .map(Map.Entry::getValue).map(Object::toString).collect(Collectors.joining("\n")) ); } String str = getConfigs().getString(config, "renameToPrefix", "").trim(); this.renameToPrefix = str.length() > 0 ? str : null; validateArguments(); }
public LoadSolr(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); this.loader = locator.getLoader(); Config boostsConfig = getConfigs().getConfig(config, "boosts", ConfigFactory.empty()); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(boostsConfig)) { String fieldName = entry.getKey(); float boost = Float.parseFloat(entry.getValue().toString().trim()); boosts.put(fieldName, boost); } validateArguments(); this.elapsedTime = getTimer(Metrics.ELAPSED_TIME); }
public LoadSolr(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); Config solrLocatorConfig = getConfigs().getConfig(config, SOLR_LOCATOR_PARAM); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); this.loader = locator.getLoader(); Config boostsConfig = getConfigs().getConfig(config, "boosts", ConfigFactory.empty()); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(boostsConfig)) { String fieldName = entry.getKey(); float boost = Float.parseFloat(entry.getValue().toString().trim()); boosts.put(fieldName, boost); } this.isDryRun = context.getTypedSettings().getBoolean(TypedSettings.DRY_RUN_SETTING_NAME, false); validateArguments(); this.elapsedTime = getTimer(Metrics.ELAPSED_TIME); }
private PatternMetricFilter(Config config) { Configs configs = new Configs(); Config includesConfig = configs.getConfig(config, "includes", null); if (includesConfig == null) { includes.add(new ExpressionPair(new MatchAllExpression(), new MatchAllExpression())); } else { for (Map.Entry<String, Object> entry : new Configs().getEntrySet(includesConfig)) { includes.add(parseExpressionPair(entry.getKey(), entry.getValue().toString(), includesConfig)); } } Config excludesConfig = configs.getConfig(config, "excludes", null); if (excludesConfig != null) { for (Map.Entry<String, Object> entry : new Configs().getEntrySet(excludesConfig)) { excludes.add(parseExpressionPair(entry.getKey(), entry.getValue().toString(), excludesConfig)); } } configs.validateArguments(config); }
public Translate(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.fieldName = getConfigs().getString(config, "field"); Config dict = getConfigs().getConfig(config, "dictionary"); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(dict)) { dictionary.put(entry.getKey(), entry.getValue()); } this.fallback = getConfigs().getString(config, "fallback", null); validateArguments(); }
private PatternMetricFilter(Config config) { Configs configs = new Configs(); Config includesConfig = configs.getConfig(config, "includes", null); if (includesConfig == null) { includes.add(new ExpressionPair(new MatchAllExpression(), new MatchAllExpression())); } else { for (Map.Entry<String, Object> entry : new Configs().getEntrySet(includesConfig)) { includes.add(parseExpressionPair(entry.getKey(), entry.getValue().toString(), includesConfig)); } } Config excludesConfig = configs.getConfig(config, "excludes", null); if (excludesConfig != null) { for (Map.Entry<String, Object> entry : new Configs().getEntrySet(excludesConfig)) { excludes.add(parseExpressionPair(entry.getKey(), entry.getValue().toString(), excludesConfig)); } } configs.validateArguments(config); }
public Translate(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.fieldName = getConfigs().getString(config, "field"); Config dict = getConfigs().getConfig(config, "dictionary"); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(dict)) { dictionary.put(entry.getKey(), entry.getValue()); } this.fallback = getConfigs().getString(config, "fallback", null); validateArguments(); }
protected Rename(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); remove = getConfigs().getBoolean(config, CONF_REMOVE, DEFAULT_REMOVE); Config paths = getConfigs().getConfig(config, CONF_FIELDS); entries = paths.entrySet(); }
protected Rename(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); remove = getConfigs().getBoolean(config, CONF_REMOVE, DEFAULT_REMOVE); Config paths = getConfigs().getConfig(config, CONF_FIELDS); entries = paths.entrySet(); }
public LoadSolr(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); Config solrLocatorConfig = getConfigs().getConfig(config, SOLR_LOCATOR_PARAM); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); RetryPolicyFactory retryPolicyFactory = parseRetryPolicyFactory( getConfigs().getConfig(config, "retryPolicy", null)); this.loader = locator.getLoader(retryPolicyFactory, new CodahaleMetricsFacade(context.getMetricRegistry())); Config boostsConfig = getConfigs().getConfig(config, "boosts", ConfigFactory.empty()); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(boostsConfig)) { String fieldName = entry.getKey(); float boost = Float.parseFloat(entry.getValue().toString().trim()); boosts.put(fieldName, boost); } this.rateLimiter = RateLimiter.create(getConfigs().getDouble(config, "maxRecordsPerSecond", Double.MAX_VALUE)); this.isDryRun = context.getTypedSettings().getBoolean(TypedSettings.DRY_RUN_SETTING_NAME, false); validateArguments(); this.elapsedTime = getTimer(Metrics.ELAPSED_TIME); }
public GenerateSolrSequenceKey(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.baseIdFieldName = getConfigs().getString(config, "baseIdField", Fields.BASE_ID); this.preserveExisting = getConfigs().getBoolean(config, "preserveExisting", true); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); IndexSchema schema = locator.getIndexSchema(); SchemaField uniqueKey = schema.getUniqueKeyField(); uniqueKeyName = uniqueKey == null ? null : uniqueKey.getName(); String tmpIdPrefix = getConfigs().getString(config, "idPrefix", null); // for load testing only Random tmpRandomIdPrefx = null; if ("random".equals(tmpIdPrefix)) { // for load testing only tmpRandomIdPrefx = new Random(new SecureRandom().nextLong()); tmpIdPrefix = null; } idPrefix = tmpIdPrefix; randomIdPrefix = tmpRandomIdPrefx; validateArguments(); }
public GenerateSolrSequenceKey(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.baseIdFieldName = getConfigs().getString(config, "baseIdField", Fields.BASE_ID); this.preserveExisting = getConfigs().getBoolean(config, "preserveExisting", true); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); IndexSchema schema = locator.getIndexSchema(); SchemaField uniqueKey = schema.getUniqueKeyField(); uniqueKeyName = uniqueKey == null ? null : uniqueKey.getName(); String tmpIdPrefix = getConfigs().getString(config, "idPrefix", null); // for load testing only Random tmpRandomIdPrefx = null; if ("random".equals(tmpIdPrefix)) { // for load testing only tmpRandomIdPrefx = new Random(new SecureRandom().nextLong()); tmpIdPrefix = null; } idPrefix = tmpIdPrefix; randomIdPrefix = tmpRandomIdPrefx; validateArguments(); }
public RetryPolicyFactory parse(Config retryPolicyConfig) { final Config boundedExponentialBackoffRetryConfig = new Configs().getConfig(retryPolicyConfig, BOUNDED_EXPONENTIAL_BACKOFF_RETRY_NAME); final long baseSleepTime = new Configs() .getNanoseconds(boundedExponentialBackoffRetryConfig, "baseSleepTime", TimeUnit.MILLISECONDS.toNanos(125)); final long maxSleepTime = new Configs() .getNanoseconds(boundedExponentialBackoffRetryConfig, "maxSleepTime", TimeUnit.SECONDS.toNanos(10)); final int maxRetries = new Configs() .getInt(boundedExponentialBackoffRetryConfig, "maxRetries", 100); final long maxElapsedTime = new Configs() .getNanoseconds(boundedExponentialBackoffRetryConfig, "maxElapsedTime", TimeUnit.SECONDS.toNanos(3 * 60)); LOG.debug("Parsed retry policy BoundedExponentialBackoffRetry with " + "baseSleepTime:{}, maxSleepTime:{}, maxRetries:{}, maxElapsedTime:{}", new Object[]{baseSleepTime, maxSleepTime, maxRetries, maxElapsedTime}); if (maxRetries <= 0 || maxElapsedTime <= 0) { return null; } return new DefaultRetryPolicyFactory( new FlexibleBoundedExponentialBackoffRetry(baseSleepTime, maxSleepTime, maxRetries, maxElapsedTime)); }
public WriteAvroToByteArray(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.format = new Validator<Format>().validateEnum( config, getConfigs().getString(config, "format", Format.container.toString()), Format.class); String codec = getConfigs().getString(config, "codec", null); if (codec == null) { this.codecFactory = null; } else { this.codecFactory = CodecFactory.fromString(codec); } Config metadataConfig = getConfigs().getConfig(config, "metadata", ConfigFactory.empty()); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(metadataConfig)) { this.metadata.put(entry.getKey(), entry.getValue().toString()); } validateArguments(); }
public WriteAvroToByteArray(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.format = new Validator<Format>().validateEnum( config, getConfigs().getString(config, "format", Format.container.toString()), Format.class); String codec = getConfigs().getString(config, "codec", null); if (codec == null) { this.codecFactory = null; } else { this.codecFactory = CodecFactory.fromString(codec); } Config metadataConfig = getConfigs().getConfig(config, "metadata", ConfigFactory.empty()); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(metadataConfig)) { this.metadata.put(entry.getKey(), entry.getValue().toString()); } validateArguments(); }
public TokenizeText(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.inputFieldName = getConfigs().getString(config, "inputField"); this.outputFieldName = getConfigs().getString(config, "outputField"); String solrFieldType = getConfigs().getString(config, "solrFieldType"); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); IndexSchema schema = locator.getIndexSchema(); FieldType fieldType = schema.getFieldTypeByName(solrFieldType); if (fieldType == null) { throw new MorphlineCompilationException("Missing Solr field type in schema.xml for name: " + solrFieldType, config); } this.analyzer = Objects.requireNonNull(fieldType.getIndexAnalyzer()); // register CharTermAttribute for later (implicit) reuse this.token = Objects.requireNonNull(analyzer.tokenStream("content", reader).addAttribute(CharTermAttribute.class)); validateArguments(); }