@Override public void closeOp(boolean abort) throws HiveException { if (!abort) { TezContext context = (TezContext) TezContext.get(); String vertexName = getConf().getVertexName(); String inputName = getConf().getInputName(); byte[] payload = null; if (hasReachedMaxSize) { initDataBuffer(true); } payload = new byte[buffer.getLength()]; System.arraycopy(buffer.getData(), 0, payload, 0, buffer.getLength()); Event event = InputInitializerEvent.create(vertexName, inputName, ByteBuffer.wrap(payload, 0, payload.length)); if (LOG.isInfoEnabled()) { LOG.info("Sending Tez event to vertex = " + vertexName + ", input = " + inputName + ". Payload size = " + payload.length); } context.getTezProcessorContext().sendEvents(Collections.singletonList(event)); } }
@Override public void closeOp(boolean abort) throws HiveException { if (!abort) { TezContext context = (TezContext) TezContext.get(); String vertexName = getConf().getVertexName(); String inputName = getConf().getInputName(); byte[] payload = null; if (hasReachedMaxSize) { initDataBuffer(true); } payload = new byte[buffer.getLength()]; System.arraycopy(buffer.getData(), 0, payload, 0, buffer.getLength()); Event event = InputInitializerEvent.create(vertexName, inputName, ByteBuffer.wrap(payload, 0, payload.length)); if (isLogInfoEnabled) { LOG.info("Sending Tez event to vertex = " + vertexName + ", input = " + inputName + ". Payload size = " + payload.length); } context.getTezProcessorContext().sendEvents(Collections.singletonList(event)); } }
@Test(timeout = 20000) public void testMissingEvent() throws InterruptedException, IOException, HiveException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(1).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 1)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.processVertex("v1"); Thread.sleep(3000l); // The pruner should not have completed. assertFalse(pruneRunnable.ended.get()); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
@Test(timeout = 5000) public void testSingleSourceOrdering1() throws InterruptedException, IOException, HiveException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(1).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 1)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.addEvent(event); pruner.processVertex("v1"); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
@Test(timeout = 5000) public void testSingleSourceOrdering2() throws InterruptedException, IOException, HiveException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(1).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 1)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.processVertex("v1"); pruner.addEvent(event); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
@Test(timeout = 5000, expected = IllegalStateException.class) public void testExtraEvents() throws InterruptedException, IOException, HiveException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(1).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 1)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.addEvent(event); pruner.addEvent(event); pruner.processVertex("v1"); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
@Test(timeout = 5000) public void testSingleSourceMultipleFiltersOrdering2() throws InterruptedException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(2).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 2)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.processVertex("v1"); pruner.addEvent(event); pruner.addEvent(event); pruner.addEvent(event); pruner.addEvent(event); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
@Test(timeout = 5000) public void testSingleSourceMultipleFiltersOrdering1() throws InterruptedException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(2).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 2)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.addEvent(event); pruner.addEvent(event); pruner.addEvent(event); pruner.addEvent(event); pruner.processVertex("v1"); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
public static InputInitializerEvent convertRootInputInitializerEventFromProto( EventProtos.RootInputInitializerEventProto proto) { InputInitializerEvent event = InputInitializerEvent.create(proto.getTargetVertexName(), proto.getTargetInputName(), (proto.hasUserPayload() ? proto.getUserPayload().asReadOnlyByteBuffer() : null)); return event; }
@Override public void closeOp(boolean abort) throws HiveException { if (!abort) { TezContext context = (TezContext) TezContext.get(); String vertexName = getConf().getVertexName(); String inputName = getConf().getInputName(); byte[] payload = null; if (hasReachedMaxSize) { initDataBuffer(true); } payload = new byte[buffer.getLength()]; System.arraycopy(buffer.getData(), 0, payload, 0, buffer.getLength()); Event event = InputInitializerEvent.create(vertexName, inputName, ByteBuffer.wrap(payload, 0, payload.length)); if (isLogInfoEnabled) { LOG.info("Sending Tez event to vertex = " + vertexName + ", input = " + inputName + ". Payload size = " + payload.length); } context.getTezProcessorContext().sendEvents(Collections.singletonList(event)); } }