int expectedRowCountAtFlip = (int) (dictionaryMaxMemoryBytes / (bytesPerEntry * 2 * directUniquePercentage)); int expectedMaxRowCountAtFull = stripeMaxBytes / (bytesPerEntry * 2); DataSimulator simulator = new DataSimulator(stripeMaxBytes / 2, stripeMaxBytes, expectedMaxRowCountAtFull * 2, dictionaryMaxMemoryBytes, 0, directColumn, dictionaryColumn); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip); simulator.advanceToNextStateChange(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull); simulator.finalOptimize(); assertFalse(simulator.isDictionaryMemoryFull());
int expectedRowCountAtFlip = (int) (dictionaryMaxMemoryBytes / (bytesPerEntry * 2 * directUniquePercentage)); int expectedMaxRowCountAtFull = stripeMaxBytes / (bytesPerEntry * 2); DataSimulator simulator = new DataSimulator(stripeMaxBytes / 2, stripeMaxBytes, expectedMaxRowCountAtFull * 2, dictionaryMaxMemoryBytes, 0, directColumn, dictionaryColumn); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip); simulator.advanceToNextStateChange(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull); simulator.finalOptimize(); assertFalse(simulator.isDictionaryMemoryFull());
int expectedRowCountAtFlip = (int) ((dictionaryMaxMemoryBytes - DICTIONARY_MEMORY_MAX_RANGE.toBytes()) / bytesPerEntry); int expectedMaxRowCountAtFull = stripeMaxBytes / bytesPerEntry; DataSimulator simulator = new DataSimulator(0, stripeMaxBytes, expectedMaxRowCountAtFull, dictionaryMaxMemoryBytes, 0, column); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip); simulator.advanceToNextStateChange(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull); simulator.finalOptimize(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);
int expectedRowCountAtFlip = (int) (dictionaryMaxMemoryBytes / bytesPerEntry / uniquePercentage); int expectedMaxRowCountAtFull = stripeMaxBytes / bytesPerEntry; DataSimulator simulator = new DataSimulator(stripeMaxBytes / 2, stripeMaxBytes, expectedMaxRowCountAtFull * 2, dictionaryMaxMemoryBytes, 0, column); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip); simulator.advanceToNextStateChange(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull); simulator.finalOptimize(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);
int expectedRowCountAtFlip = (int) (dictionaryMaxMemoryBytes / bytesPerEntry / uniquePercentage); int expectedMaxRowCountAtFull = stripeMaxBytes / bytesPerEntry; DataSimulator simulator = new DataSimulator(stripeMaxBytes / 2, stripeMaxBytes, expectedMaxRowCountAtFull * 2, dictionaryMaxMemoryBytes, 0, column); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip); simulator.advanceToNextStateChange(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull); simulator.finalOptimize(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);
int expectedRowCountAtFlip = (int) ((dictionaryMaxMemoryBytes - DICTIONARY_MEMORY_MAX_RANGE.toBytes()) / bytesPerEntry); int expectedMaxRowCountAtFull = stripeMaxBytes / bytesPerEntry; DataSimulator simulator = new DataSimulator(0, stripeMaxBytes, expectedMaxRowCountAtFull, dictionaryMaxMemoryBytes, 0, column); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip); simulator.advanceToNextStateChange(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull); simulator.finalOptimize(); assertFalse(simulator.isDictionaryMemoryFull()); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);
@Test public void testSingleDictionaryColumnByteLimit() { int bytesPerEntry = 1024; int dictionaryEntries = 1024; TestDictionaryColumn column = dictionaryColumn(bytesPerEntry, dictionaryEntries); // construct a simulator that will hit the row limit first int stripeMaxBytes = megabytes(100); int bytesPerRow = estimateIndexBytesPerValue(dictionaryEntries); int expectedMaxRowCount = stripeMaxBytes / bytesPerRow; DataSimulator simulator = new DataSimulator(0, stripeMaxBytes, expectedMaxRowCount * 10, megabytes(16), 0, column); for (int loop = 0; loop < 3; loop++) { assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); // since there dictionary columns is only 1 MB, the simulator should advance until the strip is full assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertLessThan(simulator.getRowCount(), expectedMaxRowCount); simulator.finalOptimize(); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertLessThan(simulator.getRowCount(), expectedMaxRowCount); simulator.reset(); } }
int dictionaryMaxMemoryBytesLow = dictionaryMaxMemoryBytes - (int) DICTIONARY_MEMORY_MAX_RANGE.toBytes(); int expectedMaxRowCount = (int) (dictionaryMaxMemoryBytesLow / bytesPerEntry / uniquePercentage); DataSimulator simulator = new DataSimulator(0, stripeMaxBytes, expectedMaxRowCount * 2, dictionaryMaxMemoryBytes, 0, column); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); assertTrue(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); simulator.finalOptimize(); assertTrue(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); simulator.reset();
DataSimulator simulator = new DataSimulator(stripeMaxBytes / 2, stripeMaxBytes, expectedMaxRowCount * 2, dictionaryMaxMemoryBytes, 0, column); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); assertTrue(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); simulator.finalOptimize(); assertTrue(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); simulator.reset();
DataSimulator simulator = new DataSimulator(stripeMaxBytes / 2, stripeMaxBytes, expectedMaxRowCount * 2, dictionaryMaxMemoryBytes, 0, column); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); assertTrue(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); simulator.finalOptimize(); assertTrue(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); simulator.reset();
int dictionaryMaxMemoryBytesLow = dictionaryMaxMemoryBytes - (int) DICTIONARY_MEMORY_MAX_RANGE.toBytes(); int expectedMaxRowCount = (int) (dictionaryMaxMemoryBytesLow / bytesPerEntry / uniquePercentage); DataSimulator simulator = new DataSimulator(0, stripeMaxBytes, expectedMaxRowCount * 2, dictionaryMaxMemoryBytes, 0, column); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); assertTrue(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); simulator.finalOptimize(); assertTrue(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); simulator.reset();
@Test public void testSingleDictionaryColumnByteLimit() { int bytesPerEntry = 1024; int dictionaryEntries = 1024; TestDictionaryColumn column = dictionaryColumn(bytesPerEntry, dictionaryEntries); // construct a simulator that will hit the row limit first int stripeMaxBytes = megabytes(100); int bytesPerRow = estimateIndexBytesPerValue(dictionaryEntries); int expectedMaxRowCount = stripeMaxBytes / bytesPerRow; DataSimulator simulator = new DataSimulator(0, stripeMaxBytes, expectedMaxRowCount * 10, megabytes(16), 0, column); for (int loop = 0; loop < 3; loop++) { assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); // since there dictionary columns is only 1 MB, the simulator should advance until the strip is full assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertLessThan(simulator.getRowCount(), expectedMaxRowCount); simulator.finalOptimize(); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertLessThan(simulator.getRowCount(), expectedMaxRowCount); simulator.reset(); } }
@Test public void testNoDictionariesBytesLimit() { int stripeMaxBytes = megabytes(100); int bytesPerRow = 1024; int expectedMaxRowCount = stripeMaxBytes / bytesPerRow; DataSimulator simulator = new DataSimulator(0, stripeMaxBytes, expectedMaxRowCount * 2, megabytes(16), bytesPerRow); for (int loop = 0; loop < 3; loop++) { assertFalse(simulator.isDictionaryMemoryFull()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); // since there are no dictionary columns, the simulator should advance until the strip is full assertFalse(simulator.isDictionaryMemoryFull()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); simulator.finalOptimize(); assertFalse(simulator.isDictionaryMemoryFull()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); simulator.reset(); } }
@Test public void testNoDictionariesBytesLimit() { int stripeMaxBytes = megabytes(100); int bytesPerRow = 1024; int expectedMaxRowCount = stripeMaxBytes / bytesPerRow; DataSimulator simulator = new DataSimulator(0, stripeMaxBytes, expectedMaxRowCount * 2, megabytes(16), bytesPerRow); for (int loop = 0; loop < 3; loop++) { assertFalse(simulator.isDictionaryMemoryFull()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); // since there are no dictionary columns, the simulator should advance until the strip is full assertFalse(simulator.isDictionaryMemoryFull()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); simulator.finalOptimize(); assertFalse(simulator.isDictionaryMemoryFull()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); simulator.reset(); } }
public void finalOptimize() { optimizer.finalOptimize(toIntExact(getBufferedBytes())); }
public boolean isDictionaryMemoryFull() { return optimizer.isFull(getBufferedBytes()); }
public void finalOptimize() { optimizer.finalOptimize(toIntExact(getBufferedBytes())); }
public boolean isDictionaryMemoryFull() { return optimizer.isFull(getBufferedBytes()); }
public void advanceToNextStateChange() { List<Boolean> directColumnFlags = getDirectColumnFlags(); while (!optimizer.isFull(getBufferedBytes()) && getBufferedBytes() < stripeMaxBytes && getRowCount() < stripeMaxRowCount && directColumnFlags.equals(getDirectColumnFlags())) { rowCount += 1024; for (TestDictionaryColumn dictionaryColumn : dictionaryColumns) { dictionaryColumn.advanceTo(rowCount); } optimizer.optimize(toIntExact(getBufferedBytes()), getRowCount()); } }
public void advanceToNextStateChange() { List<Boolean> directColumnFlags = getDirectColumnFlags(); while (!optimizer.isFull(getBufferedBytes()) && getBufferedBytes() < stripeMaxBytes && getRowCount() < stripeMaxRowCount && directColumnFlags.equals(getDirectColumnFlags())) { rowCount += 1024; for (TestDictionaryColumn dictionaryColumn : dictionaryColumns) { dictionaryColumn.advanceTo(rowCount); } optimizer.optimize(toIntExact(getBufferedBytes()), getRowCount()); } }