/** * Construct based on an existing {@code IndexerConf}. All initial settings are * copied from the supplied configuration * * @param indexerConf indexer configuration containing default values */ public IndexerConfBuilder(IndexerConf indexerConf) { this.table = indexerConf.getTable(); this.uniqueKeyField = indexerConf.getUniqueKeyField(); this.rowField = indexerConf.getRowField(); this.columnFamilyField = indexerConf.getColumnFamilyField(); this.tableNameField = indexerConf.getTableNameField(); this.mapperClass = indexerConf.getMapperClass(); this.uniqueKeyFormatterClass = indexerConf.getUniqueKeyFormatterClass(); this.rowReadMode = indexerConf.getRowReadMode(); this.mappingType = indexerConf.getMappingType(); this.fieldDefinitions = Lists.newArrayList(indexerConf.getFieldDefinitions()); this.documentExtractDefinitions = Lists.newArrayList(indexerConf.getDocumentExtractDefinitions()); this.globalParams = Maps.newHashMap(indexerConf.getGlobalParams()); }
private static String createTableValue(IndexerConf conf) { if (conf.tableNameIsRegex()) { return String.format("regex:%s", conf.getTable()); } else { return conf.getTable(); } }
public IndexerConf build() { checkNotNull(table, "table name"); IndexerConf conf = new IndexerConf(table); conf.setMappingType(mappingType != null ? mappingType : IndexerConf.DEFAULT_MAPPING_TYPE); conf.setRowReadMode(rowReadMode != null ? rowReadMode : IndexerConf.DEFAULT_ROW_READ_MODE); conf.setUniqueKeyField(uniqueKeyField != null ? uniqueKeyField : IndexerConf.DEFAULT_UNIQUE_KEY_FIELD); conf.setRowField(rowField); conf.setColumnFamilyField(columnFamilyField); conf.setTableNameField(tableNameField); conf.setMapperClass(mapperClass); conf.setUniqueKeyFormatterClass(uniqueKeyFormatterClass != null ? uniqueKeyFormatterClass : IndexerConf.DEFAULT_UNIQUE_KEY_FORMATTER); conf.setFieldDefinitions(fieldDefinitions); conf.setDocumentExtractDefinitions(documentExtractDefinitions); conf.setGlobalParams(globalParams); return conf; } }
if (solrMode.equals("cloud")) { int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(hbaseConf); solrWriter = new DirectSolrInputDocumentWriter(indexerDef.getName(), createCloudSolrClient(connectionParams, indexerConf.getUniqueKeyField(), zkSessionTimeout)); } else if (solrMode.equals("classic")) { connectionManager = new PoolingClientConnectionManager(); Indexer indexer = Indexer.createIndexer(indexerDef.getName(), indexerConf, indexerConf.getTable(), mapper, htablePool, sharder, solrWriter); IndexingEventListener eventListener = new IndexingEventListener( indexer, indexerConf.getTable(), indexerConf.tableNameIsRegex()); indexerDef.getSubscriptionTimestamp(), eventListener, threads, hostName, zk, hbaseConf, null, TableNamePredicates.getTableNamePredicate(indexerConf.getTable(), indexerConf.tableNameIsRegex()));
if (hbaseTableName != null) { tableNames.add(Bytes.toBytesBinary(hbaseTableName)); } else if (indexerConf.tableNameIsRegex()) { HTableDescriptor[] tables; try { Admin admin = getHbaseAdmin(); tables = admin.listTables(indexerConf.getTable()); } catch (IOException e) { throw new RuntimeException("Error occurred fetching hbase tables", e); tableNames.add(Bytes.toBytesBinary(indexerConf.getTable())); if (indexerConf.getMappingType() == MappingType.ROW) { MorphlineClasspathUtil.setupJavaCompilerClasspath();
@Override protected void calculateIndexUpdates(List<RowData> rowDataList, SolrUpdateCollector updateCollector) throws IOException { Map<String, KeyValue> idToKeyValue = calculateUniqueEvents(rowDataList); for (Entry<String, KeyValue> idToKvEntry : idToKeyValue.entrySet()) { String documentId = idToKvEntry.getKey(); KeyValue keyValue = idToKvEntry.getValue(); if (CellUtil.isDelete(keyValue)) { handleDelete(documentId, keyValue, updateCollector, uniqueKeyFormatter); } else { Result result = Result.create(Collections.<Cell>singletonList(keyValue)); SolrUpdateWriter updateWriter = new RowAndFamilyAddingSolrUpdateWriter( conf.getRowField(), conf.getColumnFamilyField(), uniqueKeyFormatter, keyValue, new IdAddingSolrUpdateWriter( conf.getUniqueKeyField(), documentId, conf.getTableNameField(), tableName, updateCollector)); mapper.map(result, updateWriter); } } }
if (conf.getRowReadMode() == RowReadMode.DYNAMIC) { if (!mapper.containsRequiredData(result)) { result = readRow(rowData); conf.getUniqueKeyField(), documentId, conf.getTableNameField(), tableName, updateCollector);
Map<String, String> params = indexerConf.getGlobalParams(); if (morphlineFile != null) { params.put(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, morphlineFile); if (indexerConf.getRowReadMode() != RowReadMode.NEVER) { LOG.warn("Changing row read mode from " + indexerConf.getRowReadMode() + " to " + RowReadMode.NEVER); indexerConf = new IndexerConfBuilder(indexerConf).rowReadMode(RowReadMode.NEVER).build(); indexerConf.setGlobalParams(params);
/** * Instantiate a ResultToSolrMapper based on a configuration supplied through an input stream. * * @param indexName name of the index for which the mapper is to be created * @param indexerConf configuration containing the index definition * @return configured ResultToSolrMapper */ public static ResultToSolrMapper createResultToSolrMapper(String indexName, IndexerConf indexerConf) { ResultToSolrMapper mapper = null; try { if (indexerConf.getMapperClass().equals(DefaultResultToSolrMapper.class)) { // FIXME: this is cheating. Knowledge about mapper implementations should be handled by IndexerComponentFactory mapper = new DefaultResultToSolrMapper(indexName, indexerConf.getFieldDefinitions(), indexerConf.getDocumentExtractDefinitions()); } else { mapper = indexerConf.getMapperClass().newInstance(); ConfigureUtil.configure(mapper, indexerConf.getGlobalParams()); } } catch (Exception e) { LOG.error("Error instantiating ResultToSolrMapper for " + indexName, e); throw new RuntimeException(e); } return mapper; }
IndexerConf indexerConf = factory.createIndexerConf(); Map<String, String> params = indexerConf.getGlobalParams(); String morphlineFile = params.get(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM); if (hbaseIndexingOpts.morphlineFile != null) { String uniqueKeyField = indexerConf.getUniqueKeyField(); Preconditions.checkNotNull(uniqueKeyField); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(hbaseIndexingOpts.zkHost).build();
private String fetchIndexerTableName(String indexerName) throws Exception{ // best effort since this could be a pattern ... IndexerDefinition indexerDefinition = get(indexerName); IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexerDefinition.getIndexerComponentFactory(), new ByteArrayInputStream(indexerDefinition.getConfiguration()), indexerDefinition.getConnectionParams()); String tableName = factory.createIndexerConf().getTable(); // TODO we should fail if the table does not exist return tableName; }
private void applyMorphLineParams(IndexerConf indexerConf) { Map<String, String> params = indexerConf.getGlobalParams(); if (morphlineFile != null) { params.put( MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, morphlineFile.getPath()); } if (morphlineId != null) { params.put( MorphlineResultToSolrMapper.MORPHLINE_ID_PARAM, morphlineId); } for (Map.Entry<String, String> entry : conf) { if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_VARIABLE_PARAM + ".")) { params.put(entry.getKey(), entry.getValue()); } if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_FIELD_PARAM + ".")) { params.put(entry.getKey(), entry.getValue()); } } indexerConf.setGlobalParams(params); }
IndexerConf indexerConf = factory.createIndexerConf(); if (indexerConf.getRowReadMode() != RowReadMode.NEVER) { LOG.warn("Changing row read mode from " + indexerConf.getRowReadMode() + " to " + RowReadMode.NEVER); indexerConf = new IndexerConfBuilder(indexerConf).rowReadMode(RowReadMode.NEVER).build(); Map<String, String> params = indexerConf.getGlobalParams(); if (indexingOpts.morphlineFile != null) { params.put(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, indexingOpts.morphlineFile.getPath());
Indexer(String indexerName, IndexerConf conf, String tableName, ResultToSolrMapper mapper, Sharder sharder, SolrInputDocumentWriter solrWriter) { this.indexerName = indexerName; this.conf = conf; this.tableName = tableName; this.mapper = mapper; try { this.uniqueKeyFormatter = conf.getUniqueKeyFormatterClass().newInstance(); } catch (Exception e) { throw new RuntimeException("Problem instantiating the UniqueKeyFormatter.", e); } ConfigureUtil.configure(uniqueKeyFormatter, conf.getGlobalParams()); this.sharder = sharder; this.solrWriter = solrWriter; this.indexingTimer = Metrics.newTimer(metricName(getClass(), "Index update calculation timer", indexerName), TimeUnit.MILLISECONDS, TimeUnit.SECONDS); }
private Indexer createIndexer(String indexName, Context context, IndexerConf indexerConf, String tableName, ResultToSolrMapper mapper, Map<String, String> indexConnectionParams) throws IOException, SharderException { Configuration conf = context.getConfiguration(); if (conf.getBoolean(INDEX_DIRECT_WRITE_CONF_KEY, false)) { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { DirectSolrInputDocumentWriter writer = createCloudSolrWriter( context, indexConnectionParams, indexerConf.getUniqueKeyField()); solrDocWriter = wrapInBufferedWriter(context, writer); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, null, solrDocWriter); } else if (solrMode.equals("classic")) { DirectSolrClassicInputDocumentWriter classicSolrWriter = createClassicSolrWriter(context, indexConnectionParams); Sharder sharder = createSharder(indexConnectionParams, classicSolrWriter.getNumServers()); solrDocWriter = wrapInBufferedWriter(context, classicSolrWriter); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, sharder, solrDocWriter); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } } else { solrDocWriter = new MapReduceSolrInputDocumentWriter(context); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, null, solrDocWriter); } }
/** * Instantiate an indexer based on the given {@link IndexerConf}. */ public static Indexer createIndexer(String indexerName, IndexerConf conf, String tableName, ResultToSolrMapper mapper, Connection tablePool, Sharder sharder, SolrInputDocumentWriter solrWriter) { switch (conf.getMappingType()) { case COLUMN: return new ColumnBasedIndexer(indexerName, conf, tableName, mapper, sharder, solrWriter); case ROW: return new RowBasedIndexer(indexerName, conf, tableName, mapper, tablePool, sharder, solrWriter); default: throw new IllegalStateException("Can't determine the type of indexing to use for mapping type " + conf.getMappingType()); } }
/** * Delete all values for a single column family from Solr. */ private void deleteFamily(KeyValue deleteKeyValue, SolrUpdateCollector updateCollector, UniqueKeyFormatter uniqueKeyFormatter, byte[] tableName) { String rowField = conf.getRowField(); String cfField = conf.getColumnFamilyField(); String rowValue; String familyValue; if (uniqueKeyFormatter instanceof UniqueTableKeyFormatter) { UniqueTableKeyFormatter uniqueTableKeyFormatter = (UniqueTableKeyFormatter) uniqueKeyFormatter; rowValue = uniqueTableKeyFormatter.formatRow(CellUtil.cloneRow(deleteKeyValue), tableName); familyValue = uniqueTableKeyFormatter.formatFamily(CellUtil.cloneFamily(deleteKeyValue), tableName); } else { rowValue = uniqueKeyFormatter.formatRow(CellUtil.cloneRow(deleteKeyValue)); familyValue = uniqueKeyFormatter.formatFamily(CellUtil.cloneFamily(deleteKeyValue)); } if (rowField != null && cfField != null) { updateCollector.deleteByQuery(String.format("(%s:%s)AND(%s:%s)", rowField, rowValue, cfField, familyValue)); } else { log.warn(String.format( "Can't delete row %s and family %s from Solr because row and/or family fields not included in the indexer configuration", rowValue, familyValue)); } }
IndexerConf indexerConf = factory.createIndexerConf(); Map<String, String> params = indexerConf.getGlobalParams(); String morphlineFile = params.get(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM); if (hbaseIndexingOpts.morphlineFile != null) {
if (hbaseTableName != null) { tableNames.add(Bytes.toBytesBinary(hbaseTableName)); } else if (indexerConf.tableNameIsRegex()) { HTableDescriptor[] tables; try { Admin admin = getHbaseAdmin(); tables = admin.listTables(indexerConf.getTable()); } catch (IOException e) { throw new RuntimeException("Error occurred fetching hbase tables", e); tableNames.add(Bytes.toBytesBinary(indexerConf.getTable())); if (indexerConf.getMappingType() == MappingType.ROW) { MorphlineClasspathUtil.setupJavaCompilerClasspath();
@Override protected void calculateIndexUpdates(List<RowData> rowDataList, SolrUpdateCollector updateCollector) throws IOException { Map<String, KeyValue> idToKeyValue = calculateUniqueEvents(rowDataList); for (Entry<String, KeyValue> idToKvEntry : idToKeyValue.entrySet()) { String documentId = idToKvEntry.getKey(); KeyValue keyValue = idToKvEntry.getValue(); if (CellUtil.isDelete(keyValue)) { handleDelete(documentId, keyValue, updateCollector, uniqueKeyFormatter); } else { Result result = Result.create(Collections.<Cell>singletonList(keyValue)); SolrUpdateWriter updateWriter = new RowAndFamilyAddingSolrUpdateWriter( conf.getRowField(), conf.getColumnFamilyField(), uniqueKeyFormatter, keyValue, new IdAddingSolrUpdateWriter( conf.getUniqueKeyField(), documentId, conf.getTableNameField(), tableName, updateCollector)); mapper.map(result, updateWriter); } } }