@Test public void testSingleDictionaryColumnByteLimit() { int bytesPerEntry = 1024; int dictionaryEntries = 1024; TestDictionaryColumn column = dictionaryColumn(bytesPerEntry, dictionaryEntries); // construct a simulator that will hit the row limit first int stripeMaxBytes = megabytes(100); int bytesPerRow = estimateIndexBytesPerValue(dictionaryEntries); int expectedMaxRowCount = stripeMaxBytes / bytesPerRow; DataSimulator simulator = new DataSimulator(0, stripeMaxBytes, expectedMaxRowCount * 10, megabytes(16), 0, column); for (int loop = 0; loop < 3; loop++) { assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); // since there dictionary columns is only 1 MB, the simulator should advance until the strip is full assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertLessThan(simulator.getRowCount(), expectedMaxRowCount); simulator.finalOptimize(); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertLessThan(simulator.getRowCount(), expectedMaxRowCount); simulator.reset(); } }
assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount);
assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount);
assertFalse(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); assertTrue(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip); assertTrue(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull); assertTrue(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);
assertFalse(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); assertTrue(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip); assertTrue(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull); assertTrue(directColumn.isDirect()); assertFalse(dictionaryColumn.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);
assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); assertTrue(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);
assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); assertTrue(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);
assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); assertTrue(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);
@Test public void testSingleDictionaryColumnByteLimit() { int bytesPerEntry = 1024; int dictionaryEntries = 1024; TestDictionaryColumn column = dictionaryColumn(bytesPerEntry, dictionaryEntries); // construct a simulator that will hit the row limit first int stripeMaxBytes = megabytes(100); int bytesPerRow = estimateIndexBytesPerValue(dictionaryEntries); int expectedMaxRowCount = stripeMaxBytes / bytesPerRow; DataSimulator simulator = new DataSimulator(0, stripeMaxBytes, expectedMaxRowCount * 10, megabytes(16), 0, column); for (int loop = 0; loop < 3; loop++) { assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); simulator.advanceToNextStateChange(); // since there dictionary columns is only 1 MB, the simulator should advance until the strip is full assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertLessThan(simulator.getRowCount(), expectedMaxRowCount); simulator.finalOptimize(); assertFalse(simulator.isDictionaryMemoryFull()); assertFalse(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertLessThan(simulator.getRowCount(), expectedMaxRowCount); simulator.reset(); } }
assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount);
assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount); assertFalse(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount);
assertFalse(column.isDirect()); assertEquals(simulator.getRowCount(), 0); assertEquals(simulator.getBufferedBytes(), 0); assertTrue(column.isDirect()); assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull); assertTrue(column.isDirect()); assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes); assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);