private void applyMorphLineParams(IndexerConf indexerConf) { Map<String, String> params = indexerConf.getGlobalParams(); if (morphlineFile != null) { params.put( MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, morphlineFile.getPath()); } if (morphlineId != null) { params.put( MorphlineResultToSolrMapper.MORPHLINE_ID_PARAM, morphlineId); } for (Map.Entry<String, String> entry : conf) { if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_VARIABLE_PARAM + ".")) { params.put(entry.getKey(), entry.getValue()); } if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_FIELD_PARAM + ".")) { params.put(entry.getKey(), entry.getValue()); } } indexerConf.setGlobalParams(params); }
private void applyMorphLineParams(IndexerConf indexerConf) { Map<String, String> params = indexerConf.getGlobalParams(); if (morphlineFile != null) { params.put( MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, morphlineFile.getPath()); } if (morphlineId != null) { params.put( MorphlineResultToSolrMapper.MORPHLINE_ID_PARAM, morphlineId); } for (Map.Entry<String, String> entry : conf) { if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_VARIABLE_PARAM + ".")) { params.put(entry.getKey(), entry.getValue()); } if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_FIELD_PARAM + ".")) { params.put(entry.getKey(), entry.getValue()); } } indexerConf.setGlobalParams(params); }
Indexer(String indexerName, IndexerConf conf, String tableName, ResultToSolrMapper mapper, Sharder sharder, SolrInputDocumentWriter solrWriter) { this.indexerName = indexerName; this.conf = conf; this.tableName = tableName; this.mapper = mapper; try { this.uniqueKeyFormatter = conf.getUniqueKeyFormatterClass().newInstance(); } catch (Exception e) { throw new RuntimeException("Problem instantiating the UniqueKeyFormatter.", e); } ConfigureUtil.configure(uniqueKeyFormatter, conf.getGlobalParams()); this.sharder = sharder; this.solrWriter = solrWriter; this.indexingTimer = Metrics.newTimer(metricName(getClass(), "Index update calculation timer", indexerName), TimeUnit.MILLISECONDS, TimeUnit.SECONDS); }
Indexer(String indexerName, IndexerConf conf, String tableName, ResultToSolrMapper mapper, Sharder sharder, SolrInputDocumentWriter solrWriter) { this.indexerName = indexerName; this.conf = conf; this.tableName = tableName; this.mapper = mapper; try { this.uniqueKeyFormatter = conf.getUniqueKeyFormatterClass().newInstance(); } catch (Exception e) { throw new RuntimeException("Problem instantiating the UniqueKeyFormatter.", e); } ConfigureUtil.configure(uniqueKeyFormatter, conf.getGlobalParams()); this.sharder = sharder; this.solrWriter = solrWriter; this.indexingTimer = Metrics.newTimer(metricName(getClass(), "Index update calculation timer", indexerName), TimeUnit.MILLISECONDS, TimeUnit.SECONDS); }
Map<String, String> params = indexerConf.getGlobalParams(); if (morphlineFile != null) { params.put(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, morphlineFile);
Map<String, String> params = indexerConf.getGlobalParams(); if (morphlineFile != null) { params.put(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, morphlineFile);
/** * Instantiate a ResultToSolrMapper based on a configuration supplied through an input stream. * * @param indexName name of the index for which the mapper is to be created * @param indexerConf configuration containing the index definition * @return configured ResultToSolrMapper */ public static ResultToSolrMapper createResultToSolrMapper(String indexName, IndexerConf indexerConf) { ResultToSolrMapper mapper = null; try { if (indexerConf.getMapperClass().equals(DefaultResultToSolrMapper.class)) { // FIXME: this is cheating. Knowledge about mapper implementations should be handled by IndexerComponentFactory mapper = new DefaultResultToSolrMapper(indexName, indexerConf.getFieldDefinitions(), indexerConf.getDocumentExtractDefinitions()); } else { mapper = indexerConf.getMapperClass().newInstance(); ConfigureUtil.configure(mapper, indexerConf.getGlobalParams()); } } catch (Exception e) { LOG.error("Error instantiating ResultToSolrMapper for " + indexName, e); throw new RuntimeException(e); } return mapper; }
/** * Instantiate a ResultToSolrMapper based on a configuration supplied through an input stream. * * @param indexName name of the index for which the mapper is to be created * @param indexerConf configuration containing the index definition * @return configured ResultToSolrMapper */ public static ResultToSolrMapper createResultToSolrMapper(String indexName, IndexerConf indexerConf) { ResultToSolrMapper mapper = null; try { if (indexerConf.getMapperClass().equals(DefaultResultToSolrMapper.class)) { // FIXME: this is cheating. Knowledge about mapper implementations should be handled by IndexerComponentFactory mapper = new DefaultResultToSolrMapper(indexName, indexerConf.getFieldDefinitions(), indexerConf.getDocumentExtractDefinitions()); } else { mapper = indexerConf.getMapperClass().newInstance(); ConfigureUtil.configure(mapper, indexerConf.getGlobalParams()); } } catch (Exception e) { LOG.error("Error instantiating ResultToSolrMapper for " + indexName, e); throw new RuntimeException(e); } return mapper; }
if (conf.getTableNameField() != null) indexerEl.setAttribute("table-name-field", conf.getTableNameField()); Map<String, String> params = conf.getGlobalParams(); addParams(params, indexerEl);
if (conf.getTableNameField() != null) indexerEl.setAttribute("table-name-field", conf.getTableNameField()); Map<String, String> params = conf.getGlobalParams(); addParams(params, indexerEl);
/** * Construct based on an existing {@code IndexerConf}. All initial settings are * copied from the supplied configuration * * @param indexerConf indexer configuration containing default values */ public IndexerConfBuilder(IndexerConf indexerConf) { this.table = indexerConf.getTable(); this.uniqueKeyField = indexerConf.getUniqueKeyField(); this.rowField = indexerConf.getRowField(); this.columnFamilyField = indexerConf.getColumnFamilyField(); this.tableNameField = indexerConf.getTableNameField(); this.mapperClass = indexerConf.getMapperClass(); this.uniqueKeyFormatterClass = indexerConf.getUniqueKeyFormatterClass(); this.rowReadMode = indexerConf.getRowReadMode(); this.mappingType = indexerConf.getMappingType(); this.fieldDefinitions = Lists.newArrayList(indexerConf.getFieldDefinitions()); this.documentExtractDefinitions = Lists.newArrayList(indexerConf.getDocumentExtractDefinitions()); this.globalParams = Maps.newHashMap(indexerConf.getGlobalParams()); }
/** * Construct based on an existing {@code IndexerConf}. All initial settings are * copied from the supplied configuration * * @param indexerConf indexer configuration containing default values */ public IndexerConfBuilder(IndexerConf indexerConf) { this.table = indexerConf.getTable(); this.uniqueKeyField = indexerConf.getUniqueKeyField(); this.rowField = indexerConf.getRowField(); this.columnFamilyField = indexerConf.getColumnFamilyField(); this.tableNameField = indexerConf.getTableNameField(); this.mapperClass = indexerConf.getMapperClass(); this.uniqueKeyFormatterClass = indexerConf.getUniqueKeyFormatterClass(); this.rowReadMode = indexerConf.getRowReadMode(); this.mappingType = indexerConf.getMappingType(); this.fieldDefinitions = Lists.newArrayList(indexerConf.getFieldDefinitions()); this.documentExtractDefinitions = Lists.newArrayList(indexerConf.getDocumentExtractDefinitions()); this.globalParams = Maps.newHashMap(indexerConf.getGlobalParams()); }
Map<String, String> params = indexerConf.getGlobalParams(); if (indexingOpts.morphlineFile != null) { params.put(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, indexingOpts.morphlineFile.getPath());
Map<String, String> params = indexerConf.getGlobalParams(); if (indexingOpts.morphlineFile != null) { params.put(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, indexingOpts.morphlineFile.getPath());
IndexerConf indexerConf = factory.createIndexerConf(); Map<String, String> params = indexerConf.getGlobalParams(); String morphlineFile = params.get(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM); if (hbaseIndexingOpts.morphlineFile != null) {
IndexerConf indexerConf = factory.createIndexerConf(); Map<String, String> params = indexerConf.getGlobalParams(); String morphlineFile = params.get(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM); if (hbaseIndexingOpts.morphlineFile != null) {