/** * Delete all values for a single row from Solr. */ private void deleteRow(KeyValue deleteKeyValue, SolrUpdateCollector updateCollector, UniqueKeyFormatter uniqueKeyFormatter, byte[] tableName) { String rowField = conf.getRowField(); String rowValue = uniqueKeyFormatter.formatRow(CellUtil.cloneRow(deleteKeyValue)); if (rowField != null) { updateCollector.deleteByQuery(String.format("%s:%s", rowField, rowValue)); } else { log.warn(String.format( "Can't delete row %s from Solr because row field not included in indexer configuration", rowValue)); } }
/** * Delete all values for a single row from Solr. */ private void deleteRow(KeyValue deleteKeyValue, SolrUpdateCollector updateCollector, UniqueKeyFormatter uniqueKeyFormatter, byte[] tableName) { String rowField = conf.getRowField(); String rowValue = uniqueKeyFormatter.formatRow(CellUtil.cloneRow(deleteKeyValue)); if (rowField != null) { updateCollector.deleteByQuery(String.format("%s:%s", rowField, rowValue)); } else { log.warn(String.format( "Can't delete row %s from Solr because row field not included in indexer configuration", rowValue)); } }
conf.getRowReadMode().toString().toLowerCase()); if (conf.getUniqueKeyField() != null) indexerEl.setAttribute("unique-key-field", conf.getUniqueKeyField()); if (conf.getRowField() != null) indexerEl.setAttribute("row-field", conf.getRowField()); if (conf.getColumnFamilyField() != null) indexerEl.setAttribute("column-family-field", conf.getColumnFamilyField()); if (conf.getTableNameField() != null) indexerEl.setAttribute("table-name-field", conf.getTableNameField());
conf.getRowReadMode().toString().toLowerCase()); if (conf.getUniqueKeyField() != null) indexerEl.setAttribute("unique-key-field", conf.getUniqueKeyField()); if (conf.getRowField() != null) indexerEl.setAttribute("row-field", conf.getRowField()); if (conf.getColumnFamilyField() != null) indexerEl.setAttribute("column-family-field", conf.getColumnFamilyField()); if (conf.getTableNameField() != null) indexerEl.setAttribute("table-name-field", conf.getTableNameField());
@Override protected void calculateIndexUpdates(List<RowData> rowDataList, SolrUpdateCollector updateCollector) throws IOException { Map<String, KeyValue> idToKeyValue = calculateUniqueEvents(rowDataList); for (Entry<String, KeyValue> idToKvEntry : idToKeyValue.entrySet()) { String documentId = idToKvEntry.getKey(); KeyValue keyValue = idToKvEntry.getValue(); if (CellUtil.isDelete(keyValue)) { handleDelete(documentId, keyValue, updateCollector, uniqueKeyFormatter); } else { Result result = Result.create(Collections.<Cell>singletonList(keyValue)); SolrUpdateWriter updateWriter = new RowAndFamilyAddingSolrUpdateWriter( conf.getRowField(), conf.getColumnFamilyField(), uniqueKeyFormatter, keyValue, new IdAddingSolrUpdateWriter( conf.getUniqueKeyField(), documentId, conf.getTableNameField(), tableName, updateCollector)); mapper.map(result, updateWriter); } } }
/** * Delete all values for a single column family from Solr. */ private void deleteFamily(KeyValue deleteKeyValue, SolrUpdateCollector updateCollector, UniqueKeyFormatter uniqueKeyFormatter, byte[] tableName) { String rowField = conf.getRowField(); String cfField = conf.getColumnFamilyField(); String rowValue; String familyValue; if (uniqueKeyFormatter instanceof UniqueTableKeyFormatter) { UniqueTableKeyFormatter uniqueTableKeyFormatter = (UniqueTableKeyFormatter) uniqueKeyFormatter; rowValue = uniqueTableKeyFormatter.formatRow(CellUtil.cloneRow(deleteKeyValue), tableName); familyValue = uniqueTableKeyFormatter.formatFamily(CellUtil.cloneFamily(deleteKeyValue), tableName); } else { rowValue = uniqueKeyFormatter.formatRow(CellUtil.cloneRow(deleteKeyValue)); familyValue = uniqueKeyFormatter.formatFamily(CellUtil.cloneFamily(deleteKeyValue)); } if (rowField != null && cfField != null) { updateCollector.deleteByQuery(String.format("(%s:%s)AND(%s:%s)", rowField, rowValue, cfField, familyValue)); } else { log.warn(String.format( "Can't delete row %s and family %s from Solr because row and/or family fields not included in the indexer configuration", rowValue, familyValue)); } }
/** * Construct based on an existing {@code IndexerConf}. All initial settings are * copied from the supplied configuration * * @param indexerConf indexer configuration containing default values */ public IndexerConfBuilder(IndexerConf indexerConf) { this.table = indexerConf.getTable(); this.uniqueKeyField = indexerConf.getUniqueKeyField(); this.rowField = indexerConf.getRowField(); this.columnFamilyField = indexerConf.getColumnFamilyField(); this.tableNameField = indexerConf.getTableNameField(); this.mapperClass = indexerConf.getMapperClass(); this.uniqueKeyFormatterClass = indexerConf.getUniqueKeyFormatterClass(); this.rowReadMode = indexerConf.getRowReadMode(); this.mappingType = indexerConf.getMappingType(); this.fieldDefinitions = Lists.newArrayList(indexerConf.getFieldDefinitions()); this.documentExtractDefinitions = Lists.newArrayList(indexerConf.getDocumentExtractDefinitions()); this.globalParams = Maps.newHashMap(indexerConf.getGlobalParams()); }
@Override protected void calculateIndexUpdates(List<RowData> rowDataList, SolrUpdateCollector updateCollector) throws IOException { Map<String, KeyValue> idToKeyValue = calculateUniqueEvents(rowDataList); for (Entry<String, KeyValue> idToKvEntry : idToKeyValue.entrySet()) { String documentId = idToKvEntry.getKey(); KeyValue keyValue = idToKvEntry.getValue(); if (CellUtil.isDelete(keyValue)) { handleDelete(documentId, keyValue, updateCollector, uniqueKeyFormatter); } else { Result result = Result.create(Collections.<Cell>singletonList(keyValue)); SolrUpdateWriter updateWriter = new RowAndFamilyAddingSolrUpdateWriter( conf.getRowField(), conf.getColumnFamilyField(), uniqueKeyFormatter, keyValue, new IdAddingSolrUpdateWriter( conf.getUniqueKeyField(), documentId, conf.getTableNameField(), tableName, updateCollector)); mapper.map(result, updateWriter); } } }
/** * Construct based on an existing {@code IndexerConf}. All initial settings are * copied from the supplied configuration * * @param indexerConf indexer configuration containing default values */ public IndexerConfBuilder(IndexerConf indexerConf) { this.table = indexerConf.getTable(); this.uniqueKeyField = indexerConf.getUniqueKeyField(); this.rowField = indexerConf.getRowField(); this.columnFamilyField = indexerConf.getColumnFamilyField(); this.tableNameField = indexerConf.getTableNameField(); this.mapperClass = indexerConf.getMapperClass(); this.uniqueKeyFormatterClass = indexerConf.getUniqueKeyFormatterClass(); this.rowReadMode = indexerConf.getRowReadMode(); this.mappingType = indexerConf.getMappingType(); this.fieldDefinitions = Lists.newArrayList(indexerConf.getFieldDefinitions()); this.documentExtractDefinitions = Lists.newArrayList(indexerConf.getDocumentExtractDefinitions()); this.globalParams = Maps.newHashMap(indexerConf.getGlobalParams()); }
/** * Delete all values for a single column family from Solr. */ private void deleteFamily(KeyValue deleteKeyValue, SolrUpdateCollector updateCollector, UniqueKeyFormatter uniqueKeyFormatter, byte[] tableName) { String rowField = conf.getRowField(); String cfField = conf.getColumnFamilyField(); String rowValue; String familyValue; if (uniqueKeyFormatter instanceof UniqueTableKeyFormatter) { UniqueTableKeyFormatter uniqueTableKeyFormatter = (UniqueTableKeyFormatter) uniqueKeyFormatter; rowValue = uniqueTableKeyFormatter.formatRow(CellUtil.cloneRow(deleteKeyValue), tableName); familyValue = uniqueTableKeyFormatter.formatFamily(CellUtil.cloneFamily(deleteKeyValue), tableName); } else { rowValue = uniqueKeyFormatter.formatRow(CellUtil.cloneRow(deleteKeyValue)); familyValue = uniqueKeyFormatter.formatFamily(CellUtil.cloneFamily(deleteKeyValue)); } if (rowField != null && cfField != null) { updateCollector.deleteByQuery(String.format("(%s:%s)AND(%s:%s)", rowField, rowValue, cfField, familyValue)); } else { log.warn(String.format( "Can't delete row %s and family %s from Solr because row and/or family fields not included in the indexer configuration", rowValue, familyValue)); } }