private void flushIndexToContextAndClose(BytesWritable key, IncrementalIndex index, Context context) throws IOException, InterruptedException { final List<String> dimensions = index.getDimensionNames(); Iterator<Row> rows = index.iterator(); while (rows.hasNext()) { context.progress(); Row row = rows.next(); InputRow inputRow = getInputRowFromRow(row, dimensions); // reportParseExceptions is true as any unparseable data is already handled by the mapper. InputRowSerde.SerializeResult serializeResult = InputRowSerde.toBytes(typeHelperMap, inputRow, combiningAggs); context.write( key, new BytesWritable(serializeResult.getSerializedRow()) ); } index.close(); }
@Test public void testCaseSensitivity() throws Exception { long timestamp = System.currentTimeMillis(); IncrementalIndex index = closerRule.closeLater(indexCreator.createIndex(defaultAggregatorFactories)); populateIndex(timestamp, index); Assert.assertEquals(Arrays.asList("dim1", "dim2"), index.getDimensionNames()); Assert.assertEquals(2, index.size()); final Iterator<Row> rows = index.iterator(); Row row = rows.next(); Assert.assertEquals(timestamp, row.getTimestampFromEpoch()); Assert.assertEquals(Collections.singletonList("1"), row.getDimension("dim1")); Assert.assertEquals(Collections.singletonList("2"), row.getDimension("dim2")); row = rows.next(); Assert.assertEquals(timestamp, row.getTimestampFromEpoch()); Assert.assertEquals(Collections.singletonList("3"), row.getDimension("dim1")); Assert.assertEquals(Collections.singletonList("4"), row.getDimension("dim2")); }
Assert.assertEquals(dimensionCount, index.getDimensionNames().size()); Assert.assertEquals(elementsPerThread * (isRollup ? 1 : threadCount), index.size()); Iterator<Row> iterator = index.iterator(); int curr = 0; while (iterator.hasNext()) {
Row row = index.iterator().next(); Assert.assertEquals(Lists.newArrayList("xsd", "aba", "fds", "aba"), row.getRaw("string1")); Assert.assertEquals(Lists.newArrayList("aba", "aba", "fds", "xsd"), row.getRaw("string2"));
Assert.assertEquals(2, index.size()); final Iterator<Row> rows = index.iterator(); Row row = rows.next(); Assert.assertEquals(timestamp, row.getTimestampFromEpoch());
private void flushIndexToContextAndClose(BytesWritable key, IncrementalIndex index, Context context) throws IOException, InterruptedException { final List<String> dimensions = index.getDimensionNames(); Iterator<Row> rows = index.iterator(); while (rows.hasNext()) { context.progress(); Row row = rows.next(); InputRow inputRow = getInputRowFromRow(row, dimensions); // reportParseExceptions is true as any unparseable data is already handled by the mapper. InputRowSerde.SerializeResult serializeResult = InputRowSerde.toBytes(typeHelperMap, inputRow, combiningAggs); context.write( key, new BytesWritable(serializeResult.getSerializedRow()) ); } index.close(); }