@Override public void closeOp(boolean abort) throws HiveException { if (!abort) { TezContext context = (TezContext) TezContext.get(); String vertexName = getConf().getVertexName(); String inputName = getConf().getInputName(); byte[] payload = null; if (hasReachedMaxSize) { initDataBuffer(true); } payload = new byte[buffer.getLength()]; System.arraycopy(buffer.getData(), 0, payload, 0, buffer.getLength()); Event event = InputInitializerEvent.create(vertexName, inputName, ByteBuffer.wrap(payload, 0, payload.length)); if (LOG.isInfoEnabled()) { LOG.info("Sending Tez event to vertex = " + vertexName + ", input = " + inputName + ". Payload size = " + payload.length); } context.getTezProcessorContext().sendEvents(Collections.singletonList(event)); } }
private void processEvents() throws SerDeException, IOException, InterruptedException { int eventCount = 0; while (true) { Object element = queue.take(); if (element == endOfEvents) { // we're done processing events break; } InputInitializerEvent event = (InputInitializerEvent) element; LOG.info("Input event: " + event.getTargetInputName() + ", " + event.getTargetVertexName() + ", " + (event.getUserPayload().limit() - event.getUserPayload().position())); processPayload(event.getUserPayload(), event.getSourceVertexName()); eventCount += 1; } LOG.info("Received events: " + eventCount); }
@Test(timeout = 20000) public void testMissingEvent() throws InterruptedException, IOException, HiveException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(1).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 1)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.processVertex("v1"); Thread.sleep(3000l); // The pruner should not have completed. assertFalse(pruneRunnable.ended.get()); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
public static EventProtos.RootInputInitializerEventProto convertRootInputInitializerEventToProto( InputInitializerEvent event) { EventProtos.RootInputInitializerEventProto.Builder builder = EventProtos.RootInputInitializerEventProto.newBuilder(); builder.setTargetVertexName(event.getTargetVertexName()); builder.setTargetInputName(event.getTargetInputName()); if (event.getUserPayload() != null) { builder.setUserPayload(ByteString.copyFrom(event.getUserPayload())); } return builder.build(); }
Vertex target = getDAG().getVertex(riEvent.getTargetVertexName()); Preconditions.checkArgument(target != null, "Event sent to unknown vertex: " + riEvent.getTargetVertexName()); riEvent.setSourceVertexName(tezEvent.getSourceInfo().getTaskVertexName()); if (target == this) { if (rootInputDescriptors == null || !rootInputDescriptors.containsKey(riEvent.getTargetInputName())) { throw new TezUncheckedException( "InputInitializerEvent targeted at unknown initializer on vertex " +
Preconditions.checkState(tezEvent.getEvent() instanceof InputInitializerEvent); InputInitializerEvent event = (InputInitializerEvent)tezEvent.getEvent(); Preconditions.checkState(vertex.getName().equals(event.getTargetVertexName()), "Received event for incorrect vertex"); Preconditions.checkNotNull(event.getTargetInputName(), "target input name must be set"); InitializerWrapper initializer = initializerMap.get(event.getTargetInputName()); Preconditions.checkState(initializer != null, "Received event for unknown input : " + event.getTargetInputName()); eventMap.put(initializer, tezEvent);
public void addEvent(InputInitializerEvent event) { synchronized(sourcesWaitingForEvents) { if (sourcesWaitingForEvents.contains(event.getSourceVertexName())) { ++totalEventCount; numEventsSeenPerSource.get(event.getSourceVertexName()).increment(); if(!queue.offer(event)) { throw new IllegalStateException("Queue full"); } checkForSourceCompletion(event.getSourceVertexName()); } } }
@Test(timeout = 5000) public void testSingleSourceOrdering1() throws InterruptedException, IOException, HiveException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(1).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 1)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.addEvent(event); pruner.processVertex("v1"); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
public void addEvent(InputInitializerEvent event) { synchronized(sourcesWaitingForEvents) { if (sourcesWaitingForEvents.contains(event.getSourceVertexName())) { ++totalEventCount; numEventsSeenPerSource.get(event.getSourceVertexName()).increment(); if(!queue.offer(event)) { throw new IllegalStateException("Queue full"); } checkForSourceCompletion(event.getSourceVertexName()); } } }
private void processEvents() throws SerDeException, IOException, InterruptedException { int eventCount = 0; while (true) { Object element = queue.take(); if (element == endOfEvents) { // we're done processing events break; } InputInitializerEvent event = (InputInitializerEvent) element; LOG.info("Input event: " + event.getTargetInputName() + ", " + event.getTargetVertexName() + ", " + (event.getUserPayload().limit() - event.getUserPayload().position())); processPayload(event.getUserPayload(), event.getSourceVertexName()); eventCount += 1; } LOG.info("Received events: " + eventCount); }
@Test(timeout = 5000) public void testSingleSourceOrdering2() throws InterruptedException, IOException, HiveException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(1).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 1)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.processVertex("v1"); pruner.addEvent(event); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
@Override public void closeOp(boolean abort) throws HiveException { if (!abort) { TezContext context = (TezContext) TezContext.get(); String vertexName = getConf().getVertexName(); String inputName = getConf().getInputName(); byte[] payload = null; if (hasReachedMaxSize) { initDataBuffer(true); } payload = new byte[buffer.getLength()]; System.arraycopy(buffer.getData(), 0, payload, 0, buffer.getLength()); Event event = InputInitializerEvent.create(vertexName, inputName, ByteBuffer.wrap(payload, 0, payload.length)); if (isLogInfoEnabled) { LOG.info("Sending Tez event to vertex = " + vertexName + ", input = " + inputName + ". Payload size = " + payload.length); } context.getTezProcessorContext().sendEvents(Collections.singletonList(event)); } }
public void addEvent(InputInitializerEvent event) { synchronized(sourcesWaitingForEvents) { if (sourcesWaitingForEvents.contains(event.getSourceVertexName())) { ++totalEventCount; numEventsSeenPerSource.get(event.getSourceVertexName()).increment(); queue.offer(event); checkForSourceCompletion(event.getSourceVertexName()); } } }
private void processEvents() throws SerDeException, IOException, InterruptedException { int eventCount = 0; while (true) { Object element = queue.take(); if (element == endOfEvents) { // we're done processing events break; } InputInitializerEvent event = (InputInitializerEvent) element; LOG.info("Input event: " + event.getTargetInputName() + ", " + event.getTargetVertexName() + ", " + (event.getUserPayload().limit() - event.getUserPayload().position())); processPayload(event.getUserPayload(), event.getSourceVertexName()); eventCount += 1; } LOG.info("Received events: " + eventCount); }
@Test(timeout = 5000, expected = IllegalStateException.class) public void testExtraEvents() throws InterruptedException, IOException, HiveException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(1).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 1)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.addEvent(event); pruner.addEvent(event); pruner.processVertex("v1"); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
public static InputInitializerEvent convertRootInputInitializerEventFromProto( EventProtos.RootInputInitializerEventProto proto) { InputInitializerEvent event = InputInitializerEvent.create(proto.getTargetVertexName(), proto.getTargetInputName(), (proto.hasUserPayload() ? proto.getUserPayload().asReadOnlyByteBuffer() : null)); return event; }
@Test(timeout = 5000) public void testSingleSourceMultipleFiltersOrdering2() throws InterruptedException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(2).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 2)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.processVertex("v1"); pruner.addEvent(event); pruner.addEvent(event); pruner.addEvent(event); pruner.addEvent(event); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }
@Override public void closeOp(boolean abort) throws HiveException { if (!abort) { TezContext context = (TezContext) TezContext.get(); String vertexName = getConf().getVertexName(); String inputName = getConf().getInputName(); byte[] payload = null; if (hasReachedMaxSize) { initDataBuffer(true); } payload = new byte[buffer.getLength()]; System.arraycopy(buffer.getData(), 0, payload, 0, buffer.getLength()); Event event = InputInitializerEvent.create(vertexName, inputName, ByteBuffer.wrap(payload, 0, payload.length)); if (isLogInfoEnabled) { LOG.info("Sending Tez event to vertex = " + vertexName + ", input = " + inputName + ". Payload size = " + payload.length); } context.getTezProcessorContext().sendEvents(Collections.singletonList(event)); } }
InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); eventV1.setSourceVertexName("v1"); InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); eventV2.setSourceVertexName("v2");
@Test(timeout = 5000) public void testSingleSourceMultipleFiltersOrdering1() throws InterruptedException, SerDeException { InputInitializerContext mockInitContext = mock(InputInitializerContext.class); doReturn(2).when(mockInitContext).getVertexNumTasks("v1"); MapWork mapWork = createMockMapWork(new TestSource("v1", 2)); DynamicPartitionPruner pruner = new DynamicPartitionPrunerForEventTesting(mockInitContext, mapWork); PruneRunnable pruneRunnable = new PruneRunnable(pruner); Thread t = new Thread(pruneRunnable); t.start(); try { pruneRunnable.start(); InputInitializerEvent event = InputInitializerEvent.create("FakeTarget", "TargetInput", ByteBuffer.allocate(0)); event.setSourceVertexName("v1"); pruner.addEvent(event); pruner.addEvent(event); pruner.addEvent(event); pruner.addEvent(event); pruner.processVertex("v1"); pruneRunnable.awaitEnd(); assertFalse(pruneRunnable.inError.get()); } finally { t.interrupt(); t.join(); } }