public MemoryEstimator(TableConfig tableConfig, File sampleCompletedSegment, long sampleSegmentConsumedSeconds) { _tableConfig = tableConfig; _sampleCompletedSegment = sampleCompletedSegment; _sampleSegmentConsumedSeconds = sampleSegmentConsumedSeconds; _sampleCompletedSegmentSizeBytes = FileUtils.sizeOfDirectory(_sampleCompletedSegment); try { _segmentMetadata = new SegmentMetadataImpl(_sampleCompletedSegment); } catch (Exception e) { throw new RuntimeException("Caught exception when reading segment index dir", e); } if (CollectionUtils.isNotEmpty(_tableConfig.getIndexingConfig().getNoDictionaryColumns())) { _noDictionaryColumns.addAll(_tableConfig.getIndexingConfig().getNoDictionaryColumns()); } if (CollectionUtils.isNotEmpty(_tableConfig.getIndexingConfig().getInvertedIndexColumns())) { _invertedIndexColumns.addAll(_tableConfig.getIndexingConfig().getInvertedIndexColumns()); } _avgMultiValues = getAvgMultiValues(); _tableDataDir = new File(TMP_DIR, _segmentMetadata.getTableName()); try { FileUtils.deleteDirectory(_tableDataDir); } catch (IOException e) { throw new RuntimeException("Exception in deleting directory " + _tableDataDir.getAbsolutePath(), e); } _tableDataDir.mkdir(); }
List<String> sortedColumn = _indexingConfig.getSortedColumn(); StarTreeIndexSpec starTreeIndexSpec = _indexingConfig.getStarTreeIndexSpec(); List<String> invertedIndexColumns = _indexingConfig.getInvertedIndexColumns();
Preconditions.checkNotNull(tableConfig); IndexingConfig indexingConfig = tableConfig.getIndexingConfig(); List<String> invertedIndexColumns = indexingConfig.getInvertedIndexColumns(); boolean autoGeneratedInvertedIndex = indexingConfig.isAutoGeneratedInvertedIndex();
List<String> invertedIndexColumns = indexingConfig.getInvertedIndexColumns(); if (invertedIndexColumns != null) { _invertedIndexColumns.addAll(invertedIndexColumns);
_invertedIndexCreationColumns = indexingConfig.getInvertedIndexColumns();
@Test public void testSerDe() throws IOException { IndexingConfig indexingConfig = new IndexingConfig(); indexingConfig.setLoadMode("MMAP"); indexingConfig.setAggregateMetrics(true); List<String> invertedIndexColumns = Arrays.asList("a", "b", "c"); indexingConfig.setInvertedIndexColumns(invertedIndexColumns); List<String> sortedColumn = Arrays.asList("d", "e", "f"); indexingConfig.setSortedColumn(sortedColumn); List<String> onHeapDictionaryColumns = Arrays.asList("x", "y", "z"); indexingConfig.setOnHeapDictionaryColumns(onHeapDictionaryColumns); List<String> bloomFilterColumns = Arrays.asList("a", "b"); indexingConfig.setBloomFilterColumns(bloomFilterColumns); Map<String, String> noDictionaryConfig = new HashMap<>(); noDictionaryConfig.put("a", "SNAPPY"); noDictionaryConfig.put("b", "PASS_THROUGH"); indexingConfig.setnoDictionaryConfig(noDictionaryConfig); indexingConfig = JsonUtils.stringToObject(JsonUtils.objectToString(indexingConfig), IndexingConfig.class); assertEquals(indexingConfig.getLoadMode(), "MMAP"); assertTrue(indexingConfig.isAggregateMetrics()); assertEquals(indexingConfig.getInvertedIndexColumns(), invertedIndexColumns); assertEquals(indexingConfig.getSortedColumn(), sortedColumn); assertEquals(indexingConfig.getOnHeapDictionaryColumns(), onHeapDictionaryColumns); assertEquals(indexingConfig.getBloomFilterColumns(), bloomFilterColumns); assertEquals(indexingConfig.getNoDictionaryConfig(), noDictionaryConfig); }
/** * Helper function to trigger the segment creation * * TODO: Support all kinds of indexing (no dictionary) */ private void buildSegment(String outputPath, String tableName, String segmentName, RecordReader recordReader, IndexingConfig indexingConfig) throws Exception { SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(recordReader.getSchema()); segmentGeneratorConfig.setOutDir(outputPath); segmentGeneratorConfig.setTableName(tableName); segmentGeneratorConfig.setSegmentName(segmentName); if (indexingConfig != null) { segmentGeneratorConfig.setInvertedIndexCreationColumns(indexingConfig.getInvertedIndexColumns()); if (indexingConfig.getStarTreeIndexSpec() != null) { segmentGeneratorConfig.enableStarTreeIndex(indexingConfig.getStarTreeIndexSpec()); } } SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl(); driver.init(segmentGeneratorConfig, recordReader); driver.build(); }