How to use

estimateIndexBytesPerValue

method

in

io.prestosql.orc.DictionaryCompressionOptimizer

@Override
public int getIndexBytes()
{
  checkState(!directEncoded);
  return toIntExact(estimateIndexBytesPerValue(dictionary.getEntryCount()) * getNonNullValueCount());
}

@Override
public int getIndexBytes()
{
  checkState(!directEncoded);
  return toIntExact(estimateIndexBytesPerValue(dictionary.getEntryCount()) * getNonNullValueCount());
}

@Override
public int getIndexBytes()
{
  return toIntExact(estimateIndexBytesPerValue(getDictionaryEntries()) * getNonNullValueCount());
}

@Override
public int getIndexBytes()
{
  return toIntExact(estimateIndexBytesPerValue(getDictionaryEntries()) * getNonNullValueCount());
}

public long getBufferedBytes()
{
  if (direct) {
    return (long) (rowCount * valuesPerRow * bytesPerEntry);
  }
  int dictionaryEntries = getDictionaryEntries();
  int bytesPerValue = estimateIndexBytesPerValue(dictionaryEntries);
  return (dictionaryEntries * bytesPerEntry) + (getNonNullValueCount() * bytesPerValue);
}

public long getBufferedBytes()
{
  if (direct) {
    return (long) (rowCount * valuesPerRow * bytesPerEntry);
  }
  int dictionaryEntries = getDictionaryEntries();
  int bytesPerValue = estimateIndexBytesPerValue(dictionaryEntries);
  return (dictionaryEntries * bytesPerEntry) + (getNonNullValueCount() * bytesPerValue);
}

@Test
public void testSingleDictionaryColumnByteLimit()
{
  int bytesPerEntry = 1024;
  int dictionaryEntries = 1024;
  TestDictionaryColumn column = dictionaryColumn(bytesPerEntry, dictionaryEntries);
  // construct a simulator that will hit the row limit first
  int stripeMaxBytes = megabytes(100);
  int bytesPerRow = estimateIndexBytesPerValue(dictionaryEntries);
  int expectedMaxRowCount = stripeMaxBytes / bytesPerRow;
  DataSimulator simulator = new DataSimulator(0, stripeMaxBytes, expectedMaxRowCount * 10, megabytes(16), 0, column);
  for (int loop = 0; loop < 3; loop++) {
    assertFalse(simulator.isDictionaryMemoryFull());
    assertFalse(column.isDirect());
    assertEquals(simulator.getRowCount(), 0);
    assertEquals(simulator.getBufferedBytes(), 0);
    simulator.advanceToNextStateChange();
    // since there dictionary columns is only 1 MB, the simulator should advance until the strip is full
    assertFalse(simulator.isDictionaryMemoryFull());
    assertFalse(column.isDirect());
    assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
    assertLessThan(simulator.getRowCount(), expectedMaxRowCount);
    simulator.finalOptimize();
    assertFalse(simulator.isDictionaryMemoryFull());
    assertFalse(column.isDirect());
    assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
    assertLessThan(simulator.getRowCount(), expectedMaxRowCount);
    simulator.reset();
  }
}

@Test
public void testSingleDictionaryColumnByteLimit()
{
  int bytesPerEntry = 1024;
  int dictionaryEntries = 1024;
  TestDictionaryColumn column = dictionaryColumn(bytesPerEntry, dictionaryEntries);
  // construct a simulator that will hit the row limit first
  int stripeMaxBytes = megabytes(100);
  int bytesPerRow = estimateIndexBytesPerValue(dictionaryEntries);
  int expectedMaxRowCount = stripeMaxBytes / bytesPerRow;
  DataSimulator simulator = new DataSimulator(0, stripeMaxBytes, expectedMaxRowCount * 10, megabytes(16), 0, column);
  for (int loop = 0; loop < 3; loop++) {
    assertFalse(simulator.isDictionaryMemoryFull());
    assertFalse(column.isDirect());
    assertEquals(simulator.getRowCount(), 0);
    assertEquals(simulator.getBufferedBytes(), 0);
    simulator.advanceToNextStateChange();
    // since there dictionary columns is only 1 MB, the simulator should advance until the strip is full
    assertFalse(simulator.isDictionaryMemoryFull());
    assertFalse(column.isDirect());
    assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
    assertLessThan(simulator.getRowCount(), expectedMaxRowCount);
    simulator.finalOptimize();
    assertFalse(simulator.isDictionaryMemoryFull());
    assertFalse(column.isDirect());
    assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
    assertLessThan(simulator.getRowCount(), expectedMaxRowCount);
    simulator.reset();
  }
}