final Supplier<Committer> committerSupplier = committerSupplierFromConcurrentMap(commitMetadata); Assert.assertEquals( 1, appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier) .getNumRowsInSegment() ); Assert.assertEquals( 2, appenderator.add(IDENTIFIERS.get(0), IR("2000", "bar", 2), committerSupplier) .getNumRowsInSegment() ); Assert.assertEquals( 1, appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 4), committerSupplier) .getNumRowsInSegment() ); Assert.assertEquals(IDENTIFIERS.subList(0, 2), sorted(appenderator.getSegments())); Assert.assertEquals( IDENTIFIERS.subList(0, 2), sorted( Lists.transform( segmentsAndMetadata.getSegments(), Assert.assertEquals(sorted(tester.getPushedSegments()), sorted(segmentsAndMetadata.getSegments()));
final Appenderator appenderator = tester.getAppenderator(); final ConcurrentMap<String, String> commitMetadata = new ConcurrentHashMap<>(); final Supplier<Committer> committerSupplier = committerSupplierFromConcurrentMap(commitMetadata); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); Assert.assertEquals(1, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); Assert.assertEquals(2, appenderator.getTotalRowCount()); Assert.assertEquals(0, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(2), IR("2001", "bar", 1), committerSupplier); Assert.assertEquals(1, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(2), IR("2001", "baz", 1), committerSupplier); Assert.assertEquals(2, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(2), IR("2001", "qux", 1), committerSupplier); Assert.assertEquals(3, appenderator.getTotalRowCount()); appenderator.add(IDENTIFIERS.get(2), IR("2001", "bob", 1), committerSupplier); Assert.assertEquals(4, appenderator.getTotalRowCount());
appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bar", 2), committerSupplier); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "baz", 3), committerSupplier); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "qux", 4), committerSupplier); eventCount.incrementAndGet(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 5), committerSupplier); appenderator.close();
appenderator.startJob(); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier, false); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier, false); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier, false); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "baz", 1), committerSupplier, false); Assert.assertEquals(3, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 1), committerSupplier, false); Assert.assertEquals(4, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 1), committerSupplier, false); Assert.assertEquals(5, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.persistAll(committerSupplier.get());
appenderator.startJob(); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "baz", 1), committerSupplier); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 1), committerSupplier); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 1), committerSupplier); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.persistAll(committerSupplier.get());
appenderator.startJob(); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); ); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); Assert.assertEquals( 276 + 2 * nullHandlingOverhead,
appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 2), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(1), IR("2000", "foo", 4), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001", "foo", 8), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T01", "foo", 16), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T02", "foo", 32), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T03", "foo", 64), Suppliers.ofInstance(Committers.nil()));
appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 2), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(1), IR("2000", "foo", 4), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001", "foo", 8), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T01", "foo", 16), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T02", "foo", 32), Suppliers.ofInstance(Committers.nil())); appenderator.add(IDENTIFIERS.get(2), IR("2001T03", "foo", 64), Suppliers.ofInstance(Committers.nil()));
); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(identifier, AppenderatorTest.IR("2000", "bar", 1), Suppliers.ofInstance(Committers.nil())); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(identifier, AppenderatorTest.IR("2000", "baz", 1), Suppliers.ofInstance(Committers.nil())); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.close();