if (conf.getRowReadMode() != null) indexerEl.setAttribute("read-row", conf.getRowReadMode().toString().toLowerCase()); if (conf.getUniqueKeyField() != null) indexerEl.setAttribute("unique-key-field", conf.getUniqueKeyField()); if (conf.getRowField() != null) indexerEl.setAttribute("row-field", conf.getRowField()); if (conf.getColumnFamilyField() != null) indexerEl.setAttribute("column-family-field", conf.getColumnFamilyField());
if (conf.getRowReadMode() != null) indexerEl.setAttribute("read-row", conf.getRowReadMode().toString().toLowerCase()); if (conf.getUniqueKeyField() != null) indexerEl.setAttribute("unique-key-field", conf.getUniqueKeyField()); if (conf.getRowField() != null) indexerEl.setAttribute("row-field", conf.getRowField()); if (conf.getColumnFamilyField() != null) indexerEl.setAttribute("column-family-field", conf.getColumnFamilyField());
private Indexer createIndexer(String indexName, Context context, IndexerConf indexerConf, String tableName, ResultToSolrMapper mapper, Map<String, String> indexConnectionParams) throws IOException, SharderException { Configuration conf = context.getConfiguration(); if (conf.getBoolean(INDEX_DIRECT_WRITE_CONF_KEY, false)) { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { DirectSolrInputDocumentWriter writer = createCloudSolrWriter( context, indexConnectionParams, indexerConf.getUniqueKeyField()); solrDocWriter = wrapInBufferedWriter(context, writer); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, null, solrDocWriter); } else if (solrMode.equals("classic")) { DirectSolrClassicInputDocumentWriter classicSolrWriter = createClassicSolrWriter(context, indexConnectionParams); Sharder sharder = createSharder(indexConnectionParams, classicSolrWriter.getNumServers()); solrDocWriter = wrapInBufferedWriter(context, classicSolrWriter); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, sharder, solrDocWriter); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } } else { solrDocWriter = new MapReduceSolrInputDocumentWriter(context); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, null, solrDocWriter); } }
@Override protected void calculateIndexUpdates(List<RowData> rowDataList, SolrUpdateCollector updateCollector) throws IOException { Map<String, KeyValue> idToKeyValue = calculateUniqueEvents(rowDataList); for (Entry<String, KeyValue> idToKvEntry : idToKeyValue.entrySet()) { String documentId = idToKvEntry.getKey(); KeyValue keyValue = idToKvEntry.getValue(); if (CellUtil.isDelete(keyValue)) { handleDelete(documentId, keyValue, updateCollector, uniqueKeyFormatter); } else { Result result = Result.create(Collections.<Cell>singletonList(keyValue)); SolrUpdateWriter updateWriter = new RowAndFamilyAddingSolrUpdateWriter( conf.getRowField(), conf.getColumnFamilyField(), uniqueKeyFormatter, keyValue, new IdAddingSolrUpdateWriter( conf.getUniqueKeyField(), documentId, conf.getTableNameField(), tableName, updateCollector)); mapper.map(result, updateWriter); } } }
@Override protected void calculateIndexUpdates(List<RowData> rowDataList, SolrUpdateCollector updateCollector) throws IOException { Map<String, KeyValue> idToKeyValue = calculateUniqueEvents(rowDataList); for (Entry<String, KeyValue> idToKvEntry : idToKeyValue.entrySet()) { String documentId = idToKvEntry.getKey(); KeyValue keyValue = idToKvEntry.getValue(); if (CellUtil.isDelete(keyValue)) { handleDelete(documentId, keyValue, updateCollector, uniqueKeyFormatter); } else { Result result = Result.create(Collections.<Cell>singletonList(keyValue)); SolrUpdateWriter updateWriter = new RowAndFamilyAddingSolrUpdateWriter( conf.getRowField(), conf.getColumnFamilyField(), uniqueKeyFormatter, keyValue, new IdAddingSolrUpdateWriter( conf.getUniqueKeyField(), documentId, conf.getTableNameField(), tableName, updateCollector)); mapper.map(result, updateWriter); } } }
/** * Construct based on an existing {@code IndexerConf}. All initial settings are * copied from the supplied configuration * * @param indexerConf indexer configuration containing default values */ public IndexerConfBuilder(IndexerConf indexerConf) { this.table = indexerConf.getTable(); this.uniqueKeyField = indexerConf.getUniqueKeyField(); this.rowField = indexerConf.getRowField(); this.columnFamilyField = indexerConf.getColumnFamilyField(); this.tableNameField = indexerConf.getTableNameField(); this.mapperClass = indexerConf.getMapperClass(); this.uniqueKeyFormatterClass = indexerConf.getUniqueKeyFormatterClass(); this.rowReadMode = indexerConf.getRowReadMode(); this.mappingType = indexerConf.getMappingType(); this.fieldDefinitions = Lists.newArrayList(indexerConf.getFieldDefinitions()); this.documentExtractDefinitions = Lists.newArrayList(indexerConf.getDocumentExtractDefinitions()); this.globalParams = Maps.newHashMap(indexerConf.getGlobalParams()); }
/** * Construct based on an existing {@code IndexerConf}. All initial settings are * copied from the supplied configuration * * @param indexerConf indexer configuration containing default values */ public IndexerConfBuilder(IndexerConf indexerConf) { this.table = indexerConf.getTable(); this.uniqueKeyField = indexerConf.getUniqueKeyField(); this.rowField = indexerConf.getRowField(); this.columnFamilyField = indexerConf.getColumnFamilyField(); this.tableNameField = indexerConf.getTableNameField(); this.mapperClass = indexerConf.getMapperClass(); this.uniqueKeyFormatterClass = indexerConf.getUniqueKeyFormatterClass(); this.rowReadMode = indexerConf.getRowReadMode(); this.mappingType = indexerConf.getMappingType(); this.fieldDefinitions = Lists.newArrayList(indexerConf.getFieldDefinitions()); this.documentExtractDefinitions = Lists.newArrayList(indexerConf.getDocumentExtractDefinitions()); this.globalParams = Maps.newHashMap(indexerConf.getGlobalParams()); }
String uniqueKeyField = indexerConf.getUniqueKeyField(); Preconditions.checkNotNull(uniqueKeyField); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(hbaseIndexingOpts.zkHost).build();
conf.getUniqueKeyField(), documentId, conf.getTableNameField(),
conf.getUniqueKeyField(), documentId, conf.getTableNameField(),
if (solrMode.equals("cloud")) { int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(hbaseConf); solrWriter = new DirectSolrInputDocumentWriter(indexerDef.getName(), createCloudSolrClient(connectionParams, indexerConf.getUniqueKeyField(), zkSessionTimeout)); } else if (solrMode.equals("classic")) { connectionManager = new PoolingClientConnectionManager();