/** * Creates a new data set that contains a sequence of numbers. The data set will be created in parallel, * so there is no guarantee about the oder of the elements. * * @param from The number to start at (inclusive). * @param to The number to stop at (inclusive). * @return A DataSet, containing all number in the {@code [from, to]} interval. */ public DataSource<Long> generateSequence(long from, long to) { return fromParallelCollection(new NumberSequenceIterator(from, to), BasicTypeInfo.LONG_TYPE_INFO); }
@Override public Iterator<Long> getSplit(int num, int numPartitions) { if (numPartitions < 1 || num < 0 || num >= numPartitions) { throw new IllegalArgumentException(); } return split(numPartitions)[num]; }
private static final void testSplitting(NumberSequenceIterator iter, int numSplits) { NumberSequenceIterator[] splits = iter.split(numSplits); assertEquals(numSplits, splits.length); // test start and end of range assertEquals(iter.getCurrent(), splits[0].getCurrent()); assertEquals(iter.getTo(), splits[numSplits-1].getTo()); // test continuous range for (int i = 1; i < splits.length; i++) { assertEquals(splits[i-1].getTo() + 1, splits[i].getCurrent()); } testMaxSplitDiff(splits); }
private static final void testMaxSplitDiff(NumberSequenceIterator[] iters) { long minSplitSize = Long.MAX_VALUE; long maxSplitSize = Long.MIN_VALUE; for (NumberSequenceIterator iter : iters) { long diff; if (iter.getTo() < iter.getCurrent()) { diff = 0; } else { diff = iter.getTo() - iter.getCurrent(); } if (diff < 0) { diff = Long.MAX_VALUE; } minSplitSize = Math.min(minSplitSize, diff); maxSplitSize = Math.max(maxSplitSize, diff); } assertTrue(maxSplitSize == minSplitSize || maxSplitSize-1 == minSplitSize); }
return new NumberSequenceIterator[] { new NumberSequenceIterator(current, to) }; for (; i < numWithExtra; i++) { long next = curr + elementsPerSplit + 1; iters[i] = new NumberSequenceIterator(curr, next-1); curr = next; iters[i] = new NumberSequenceIterator(curr, next-1, true); curr = next; new NumberSequenceIterator(current, current + elementsPerSplit), new NumberSequenceIterator(current + elementsPerSplit, to) };
@Override public Iterator<Long> getSplit(int num, int numPartitions) { if (numPartitions < 1 || num < 0 || num >= numPartitions) { throw new IllegalArgumentException(); } return split(numPartitions)[num]; }
return new NumberSequenceIterator[] { new NumberSequenceIterator(current, to) }; for (; i < numWithExtra; i++) { long next = curr + elementsPerSplit + 1; iters[i] = new NumberSequenceIterator(curr, next-1); curr = next; iters[i] = new NumberSequenceIterator(curr, next-1, true); curr = next; new NumberSequenceIterator(current, current + elementsPerSplit), new NumberSequenceIterator(current + elementsPerSplit, to) };
@Test public void testSplitRegular() { testSplitting(new NumberSequenceIterator(0, 10), 2); testSplitting(new NumberSequenceIterator(100, 100000), 7); testSplitting(new NumberSequenceIterator(-100, 0), 5); testSplitting(new NumberSequenceIterator(-100, 100), 3); }
@Test public void testSplittingTooSmallRanges() { testSplitting(new NumberSequenceIterator(0, 0), 2); testSplitting(new NumberSequenceIterator(-5, -5), 2); testSplitting(new NumberSequenceIterator(-5, -4), 3); testSplitting(new NumberSequenceIterator(10, 15), 10); }
@Test public void testSplittingLargeRangesBy2() { testSplitting(new NumberSequenceIterator(0, Long.MAX_VALUE), 2); testSplitting(new NumberSequenceIterator(-1000000000L, Long.MAX_VALUE), 2); testSplitting(new NumberSequenceIterator(Long.MIN_VALUE, Long.MAX_VALUE), 2); }