public boolean canAppendRow() { synchronized (hydrantLock) { return writable && currHydrant != null && currHydrant.getIndex().canAppendRow(); } }
@Override protected void reduce(final BytesWritable key, Iterable<BytesWritable> values, final Context context) throws IOException, InterruptedException { Iterator<BytesWritable> iter = values.iterator(); BytesWritable first = iter.next(); if (iter.hasNext()) { LinkedHashSet<String> dimOrder = new LinkedHashSet<>(); SortableBytes keyBytes = SortableBytes.fromBytesWritable(key); Bucket bucket = Bucket.fromGroupKey(keyBytes.getGroupKey()).lhs; IncrementalIndex index = makeIncrementalIndex(bucket, combiningAggs, config, null, null); index.add(InputRowSerde.fromBytes(typeHelperMap, first.getBytes(), aggregators)); while (iter.hasNext()) { context.progress(); InputRow value = InputRowSerde.fromBytes(typeHelperMap, iter.next().getBytes(), aggregators); if (!index.canAppendRow()) { dimOrder.addAll(index.getDimensionOrder()); log.info("current index full due to [%s]. creating new index.", index.getOutOfRowsReason()); flushIndexToContextAndClose(key, index, context); index = makeIncrementalIndex(bucket, combiningAggs, config, dimOrder, index.getColumnCapabilities()); } index.add(value); } flushIndexToContextAndClose(key, index, context); } else { context.write(key, first); } }
if (!index.canAppendRow()) { allDimensionNames.addAll(index.getDimensionOrder());
if (!index.canAppendRow()) { File tmp = tempFolder.newFolder(); toMerge.add(tmp);
public boolean canAppendRow() { synchronized (hydrantLock) { return writable && currHydrant != null && currHydrant.getIndex().canAppendRow(); } }
@Override protected void reduce( final BytesWritable key, Iterable<BytesWritable> values, final Context context ) throws IOException, InterruptedException { Iterator<BytesWritable> iter = values.iterator(); BytesWritable first = iter.next(); if (iter.hasNext()) { LinkedHashSet<String> dimOrder = Sets.newLinkedHashSet(); SortableBytes keyBytes = SortableBytes.fromBytesWritable(key); Bucket bucket = Bucket.fromGroupKey(keyBytes.getGroupKey()).lhs; IncrementalIndex index = makeIncrementalIndex(bucket, combiningAggs, config, null, null); index.add(InputRowSerde.fromBytes(typeHelperMap, first.getBytes(), aggregators)); while (iter.hasNext()) { context.progress(); InputRow value = InputRowSerde.fromBytes(typeHelperMap, iter.next().getBytes(), aggregators); if (!index.canAppendRow()) { dimOrder.addAll(index.getDimensionOrder()); log.info("current index full due to [%s]. creating new index.", index.getOutOfRowsReason()); flushIndexToContextAndClose(key, index, context); index = makeIncrementalIndex(bucket, combiningAggs, config, dimOrder, index.getColumnCapabilities()); } index.add(value); } flushIndexToContextAndClose(key, index, context); } else { context.write(key, first); } }
if (!index.canAppendRow()) { allDimensionNames.addAll(index.getDimensionOrder());