@Override public void onVertexStateUpdated(VertexStateUpdate stateUpdate) { pruner.processVertex(stateUpdate.getVertexName()); }
@Override public void onVertexStateUpdated(VertexStateUpdate stateUpdate) { pruner.processVertex(stateUpdate.getVertexName()); }
@Test(timeout = 20000) public void testMissingEvent() throws InterruptedException, IOException, HiveException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(1).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 1)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.processVertex("v1"); Thread.sleep(3000l); // The pruner should not have completed. assertFalse(pruneRunnable.ended.get()); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
@Test(timeout = 5000) public void testSingleSourceOrdering1() throws InterruptedException, IOException, HiveException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(1).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 1)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.addEvent(event); pruner.processVertex("v1"); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
@Test(timeout = 5000) public void testSingleSourceOrdering2() throws InterruptedException, IOException, HiveException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(1).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 1)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.processVertex("v1"); pruner.addEvent(event); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
@Test(timeout = 5000, expected = IllegalStateException.class) public void testExtraEvents() throws InterruptedException, IOException, HiveException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(1).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 1)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.addEvent(event); pruner.addEvent(event); pruner.processVertex("v1"); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
@Test(timeout = 5000) public void testSingleSourceMultipleFiltersOrdering1() throws InterruptedException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(2).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 2)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.addEvent(event); pruner.addEvent(event); pruner.addEvent(event); pruner.addEvent(event); pruner.processVertex("v1"); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
@Test(timeout = 5000) public void testSingleSourceMultipleFiltersOrdering2() throws InterruptedException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(2).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 2)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.processVertex("v1"); pruner.addEvent(event); pruner.addEvent(event); pruner.addEvent(event); pruner.addEvent(event); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
pruner.processVertex("v1"); pruner.addEvent(eventV1); pruner.addEvent(eventV1); pruner.addEvent(eventV2); pruner.processVertex("v2"); pruner.addEvent(eventV2); pruner.addEvent(eventV2);
pruner.processVertex("v1"); pruner.processVertex("v2"); pruner.addEvent(eventV1); pruner.addEvent(eventV1);
pruner.addEvent(eventV2); pruner.addEvent(eventV2); pruner.processVertex("v1"); pruner.processVertex("v2");
@Override public void onVertexStateUpdated(VertexStateUpdate stateUpdate) { pruner.processVertex(stateUpdate.getVertexName()); }