/** * Adds a new row. The row might correspond with another row that already exists, in which case this will * update that row instead of inserting a new one. * <p> * <p> * Calls to add() are thread safe. * <p> * * @param row the row of data to add * * @return the number of rows in the data set after adding the InputRow */ public IncrementalIndexAddResult add(InputRow row) throws IndexSizeExceededException { return add(row, false); }
@Override public void close() { index.close(); } }
@Override public int getNumRows() { return index.size(); }
@Override public List<String> getColumnNames() { List<String> columnNames = new ArrayList<>(getDimensionNames()); columnNames.addAll(getMetricNames()); return columnNames; }
@Nullable public DateTime getMinTime() { return isEmpty() ? null : DateTimes.utc(getMinTimeMillis()); }
@Nullable public DateTime getMaxTime() { return isEmpty() ? null : DateTimes.utc(getMaxTimeMillis()); }
final InputRow inputRow = index.formatRow(InputRowSerde.fromBytes(typeHelperMap, bw.getBytes(), aggregators)); int numRows = index.add(inputRow).getRowCount(); if (!index.canAppendRow()) { allDimensionNames.addAll(index.getDimensionOrder()); log.info(index.getOutOfRowsReason()); log.info( "%,d lines to %,d rows in %,d millis", config, allDimensionNames, persistIndex.getColumnCapabilities() ); startTime = System.currentTimeMillis(); allDimensionNames.addAll(index.getDimensionOrder()); if (index.isEmpty()) { throw new IAE("If you try to persist empty indexes you are going to have a bad time"); persist(index, interval, mergedBase, progressIndicator); } else { if (!index.isEmpty()) { final File finalFile = new File(baseFlushFile, "final"); persist(index, interval, finalFile, progressIndicator); index.close(); if (persistExecutor != null) {
@Test public void sameRow() throws IndexSizeExceededException { MapBasedInputRow row = new MapBasedInputRow( System.currentTimeMillis() - 1, Lists.newArrayList("billy", "joe"), ImmutableMap.of("billy", "A", "joe", "B") ); IncrementalIndex index = closerRule.closeLater(indexCreator.createIndex()); index.add(row); index.add(row); index.add(row); Assert.assertEquals(1, index.size()); } }
); index.add( new MapBasedInputRow( timestamp, index.add( new MapBasedInputRow( timestamp, Assert.assertEquals(Arrays.asList("dim1", "dim2", "dim3"), index.getDimensionNames()); Assert.assertEquals( Arrays.asList( "count_numeric_filtered" ), index.getMetricNames() ); Assert.assertEquals(2, index.size()); final Iterator<Row> rows = index.iterator(); Row row = rows.next(); Assert.assertEquals(timestamp, row.getTimestampFromEpoch());
dimOrder.addAll(oldIndex.getDimensionOrder()); oldCapabilities = oldIndex.getColumnCapabilities(); newIndex.loadDimensionIterable(dimOrder, oldCapabilities); numRowsExcludingCurrIndex.addAndGet(old.getIndex().size()); newIndex.close(); throw new ISE("finishWriting() called during swap");
@Override protected void reduce(final BytesWritable key, Iterable<BytesWritable> values, final Context context) throws IOException, InterruptedException { Iterator<BytesWritable> iter = values.iterator(); BytesWritable first = iter.next(); if (iter.hasNext()) { LinkedHashSet<String> dimOrder = new LinkedHashSet<>(); SortableBytes keyBytes = SortableBytes.fromBytesWritable(key); Bucket bucket = Bucket.fromGroupKey(keyBytes.getGroupKey()).lhs; IncrementalIndex index = makeIncrementalIndex(bucket, combiningAggs, config, null, null); index.add(InputRowSerde.fromBytes(typeHelperMap, first.getBytes(), aggregators)); while (iter.hasNext()) { context.progress(); InputRow value = InputRowSerde.fromBytes(typeHelperMap, iter.next().getBytes(), aggregators); if (!index.canAppendRow()) { dimOrder.addAll(index.getDimensionOrder()); log.info("current index full due to [%s]. creating new index.", index.getOutOfRowsReason()); flushIndexToContextAndClose(key, index, context); index = makeIncrementalIndex(bucket, combiningAggs, config, dimOrder, index.getColumnCapabilities()); } index.add(value); } flushIndexToContextAndClose(key, index, context); } else { context.write(key, first); } }
if (!index.canAppendRow()) { File tmp = tempFolder.newFolder(); toMerge.add(tmp); indexMerger.persist(index, tmp, new IndexSpec(), null); index.close(); index = new IncrementalIndex.Builder() .setIndexSchema( index.add(((StringInputRowParser) parser).parse((String) row)); } else { index.add(((List<InputRow>) parser.parseBatch(row)).get(0)); index.close();
) throws IOException if (index.isEmpty()) { throw new IAE("Trying to persist an empty index!"); final DateTime firstTimestamp = index.getMinTime(); final DateTime lastTimestamp = index.getMaxTime(); if (!(dataInterval.contains(firstTimestamp) && dataInterval.contains(lastTimestamp))) { throw new IAE( log.info("Starting persist for interval[%s], rows[%,d]", dataInterval, index.size()); return merge( Collections.singletonList( index.getMetricAggs(), outDir, indexSpec,
boolean isRollup = index.isRollup(); Assert.assertEquals(dimensionCount, index.getDimensionNames().size()); Assert.assertEquals(elementsPerThread * (isRollup ? 1 : threadCount), index.size()); Iterator<Row> iterator = index.iterator(); int curr = 0; while (iterator.hasNext()) {
private void flushIndexToContextAndClose(BytesWritable key, IncrementalIndex index, Context context) throws IOException, InterruptedException { final List<String> dimensions = index.getDimensionNames(); Iterator<Row> rows = index.iterator(); while (rows.hasNext()) { context.progress(); Row row = rows.next(); InputRow inputRow = getInputRowFromRow(row, dimensions); // reportParseExceptions is true as any unparseable data is already handled by the mapper. InputRowSerde.SerializeResult serializeResult = InputRowSerde.toBytes(typeHelperMap, inputRow, combiningAggs); context.write( key, new BytesWritable(serializeResult.getSerializedRow()) ); } index.close(); }
@Test public void testCaseSensitivity() throws Exception { long timestamp = System.currentTimeMillis(); IncrementalIndex index = closerRule.closeLater(indexCreator.createIndex(defaultAggregatorFactories)); populateIndex(timestamp, index); Assert.assertEquals(Arrays.asList("dim1", "dim2"), index.getDimensionNames()); Assert.assertEquals(2, index.size()); final Iterator<Row> rows = index.iterator(); Row row = rows.next(); Assert.assertEquals(timestamp, row.getTimestampFromEpoch()); Assert.assertEquals(Collections.singletonList("1"), row.getDimension("dim1")); Assert.assertEquals(Collections.singletonList("2"), row.getDimension("dim2")); row = rows.next(); Assert.assertEquals(timestamp, row.getTimestampFromEpoch()); Assert.assertEquals(Collections.singletonList("3"), row.getDimension("dim1")); Assert.assertEquals(Collections.singletonList("4"), row.getDimension("dim2")); }
.buildOnheap(); toPersist.getMetadata().put("key", "value"); for (InputRow event : events) { toPersist.add(event);
@Override public Interval getInterval() { return index.getInterval(); }
); try { int numRows = indexToPersist.getIndex().size(); indexToPersist.getIndex().getMetadata().putAll(metadataElems); final File persistedFile = indexMerger.persist( indexToPersist.getIndex(),
@Override public Metadata getMetadata() { return index.getMetadata(); } }