@Config("hive.orc.writer.dictionary-max-memory") public OrcFileWriterConfig setDictionaryMaxMemory(DataSize dictionaryMaxMemory) { options = options.withDictionaryMaxMemory(dictionaryMaxMemory); return this; }
private static OrcWriter createOrcFileWriter(OrcDataSink sink, List<Type> types) { List<String> columnNames = IntStream.range(0, types.size()) .mapToObj(String::valueOf) .collect(toImmutableList()); return new OrcWriter( sink, columnNames, types, ORC, LZ4, new OrcWriterOptions() .withMaxStringStatisticsLimit(new DataSize(0, BYTE)) .withStripeMinSize(new DataSize(64, MEGABYTE)) .withDictionaryMaxMemory(new DataSize(1, MEGABYTE)), ImmutableMap.of(), UTC, false, OrcWriteValidationMode.BOTH, new OrcWriterStats()); } }
.withStripeMaxSize(getOrcOptimizedWriterMaxStripeSize(session)) .withStripeMaxRowCount(getOrcOptimizedWriterMaxStripeRows(session)) .withDictionaryMaxMemory(getOrcOptimizedWriterMaxDictionaryMemory(session)) .withMaxStringStatisticsLimit(getOrcStringStatisticsLimit(session)), fileInputColumnIndexes,
.withStripeMaxRowCount(ORC_STRIPE_SIZE) .withRowGroupMaxRowCount(ORC_ROW_GROUP_SIZE) .withDictionaryMaxMemory(new DataSize(32, MEGABYTE)), ImmutableMap.of(), HIVE_STORAGE_TIME_ZONE,
.withStripeMaxRowCount(ORC_STRIPE_SIZE) .withRowGroupMaxRowCount(ORC_ROW_GROUP_SIZE) .withDictionaryMaxMemory(new DataSize(32, MEGABYTE)), ImmutableMap.of(), HIVE_STORAGE_TIME_ZONE,
.withStripeMaxRowCount(ORC_STRIPE_SIZE) .withRowGroupMaxRowCount(ORC_ROW_GROUP_SIZE) .withDictionaryMaxMemory(new DataSize(32, MEGABYTE)), ImmutableMap.of(), HIVE_STORAGE_TIME_ZONE,
.withStripeMaxRowCount(ORC_STRIPE_SIZE) .withRowGroupMaxRowCount(ORC_ROW_GROUP_SIZE) .withDictionaryMaxMemory(new DataSize(32, MEGABYTE)), ImmutableMap.of(), HIVE_STORAGE_TIME_ZONE,