public PartitionedEvent copy() { PartitionedEvent copied = new PartitionedEvent(); copied.setEvent(this.getEvent()); copied.setPartition(this.partition); copied.setPartitionKey(this.partitionKey); return copied; }
@Override public PartitionedEvent deserialize(DataInput dataInput) throws IOException { PartitionedEvent event = new PartitionedEvent(); event.setPartitionKey(dataInput.readLong()); StreamEvent streamEvent = streamEventSerializer.deserialize(dataInput); event.setEvent(streamEvent); StreamPartition partition = streamPartitionSerializer.deserialize(dataInput); partition.setStreamId(streamEvent.getStreamId()); event.setPartition(partition); return event; } }
@Override public PartitionedEvent deserialize(DataInput dataInput) throws IOException { PartitionedEvent event = new PartitionedEvent(); event.setPartitionKey(dataInput.readLong()); StreamEvent streamEvent = streamEventSerializer.deserialize(dataInput); StreamPartition partition = streamPartitionSerializer.deserialize(dataInput); event.setEvent(streamEvent); partition.setStreamId(streamEvent.getStreamId()); event.setPartition(partition); return event; }
public static PartitionedEvent createRandomOutOfTimeOrderEventGroupedByName(String streamId) { StreamEvent event = createRandomStreamEvent(streamId); event.setTimestamp(System.currentTimeMillis() + TIME_DELTA_OPTIONS[RANDOM.nextInt(TIME_DELTA_OPTIONS.length)]); return new PartitionedEvent(event, createSampleStreamGroupbyPartition(streamId, Arrays.asList("name")), event.getData()[0].hashCode()); }
public static PartitionedEvent createRandomPartitionedEvent(String streamId, long timestamp) { StreamEvent event = createRandomStreamEvent(streamId, timestamp); PartitionedEvent partitionedEvent = new PartitionedEvent(event, createSampleStreamGroupbyPartition(streamId, Arrays.asList("name")), event.getData()[0].hashCode()); return partitionedEvent; } }
public static PartitionedEvent createRandomSortedEventGroupedByName(String streamId) { StreamEvent event = createRandomStreamEvent(streamId); event.setTimestamp(System.currentTimeMillis()); return new PartitionedEvent(event, createSampleStreamGroupbyPartition(streamId, Arrays.asList("name")), event.getData()[0].hashCode()); }
private PartitionedEvent readPartitionedEvent(DataInput2 in) throws IOException { PartitionedEvent event = new PartitionedEvent(); event.setPartitionKey(in.unpackLong()); int partitionHashCode = in.unpackInt(); if (partitionHashCode != 0 && hashCodePartitionDict.containsKey(partitionHashCode)) { event.setPartition(hashCodePartitionDict.get(partitionHashCode)); } int eventBytesLen = in.unpackInt(); if (eventBytesLen > 0) { byte[] eventBytes = new byte[eventBytesLen]; in.readFully(eventBytes); event.setEvent((StreamEvent) SerializableUtils.deserializeFromCompressedByteArray(eventBytes, "Deserialize event from bytes")); } return event; }
public static PartitionedEvent createPartitionedEventGroupedByName(String streamId, long timestamp) { StreamEvent event = createRandomStreamEvent(streamId); event.setTimestamp(timestamp); return new PartitionedEvent(event, createSampleStreamGroupbyPartition(streamId, Arrays.asList("name")), event.getData()[0].hashCode()); }
PartitionedEvent pEvent = new PartitionedEvent(event, groupingStrategy.partition, hash); if (this.serializer == null) { delegate.emit(sid, Collections.singletonList(pEvent), newMessageId);
private void sendData(AlertBolt alertBolt, Map<String, StreamDefinition> definitionMap, PolicyDefinition policyDefinition) { StreamDefinition definition = definitionMap.get("perfmon_latency_stream"); long base = System.currentTimeMillis(); for (int i = 0; i < 2; i++) { long time = base + i * 1000; Map<String, Object> mapdata = new HashMap<>(); mapdata.put("host", "host-1"); mapdata.put("timestamp", time); mapdata.put("metric", "perfmon_latency"); mapdata.put("pool", "raptor"); mapdata.put("value", 1000.0 + i * 1000.0); mapdata.put("colo", "phx"); StreamEvent event = StreamEvent.builder().timestamep(time).attributes(mapdata, definition).build(); PartitionedEvent pEvent = new PartitionedEvent(event, policyDefinition.getPartitionSpec().get(0), 1); GeneralTopologyContext mock = Mockito.mock(GeneralTopologyContext.class); Mockito.when(mock.getComponentId(1)).thenReturn("taskId"); Mockito.when(mock.getComponentOutputFields("taskId", "test-stream-id")).thenReturn(new Fields(AlertConstants.FIELD_0)); TupleImpl ti = new TupleImpl(mock, Collections.singletonList(pEvent), 1, "test-stream-id"); alertBolt.execute(ti); } }
String targetStreamId = StreamIdConversion.generateStreamIdBetween(sourceId, streamRoute.getTargetComponentId()); try { PartitionedEvent emittedEvent = new PartitionedEvent(newEvent, partition, streamRoute.getPartitionKey());
private Tuple createTuple(AlertBolt bolt, String version) throws IOException { GeneralTopologyContext context = mock(GeneralTopologyContext.class); int taskId = 1; when(context.getComponentId(taskId)).thenReturn("comp1"); when(context.getComponentOutputFields("comp1", TEST_STREAM)).thenReturn(new Fields("f0")); // case 1: bolt prepared but metadata not initialized (no bolt.onAlertBoltSpecChange) PartitionedEvent pe = new PartitionedEvent(); pe.setPartitionKey(1); pe.setPartition(createPartition()); StreamEvent streamEvent = new StreamEvent(); streamEvent.setStreamId(TEST_STREAM); streamEvent.setTimestamp(System.currentTimeMillis()); streamEvent.setMetaVersion(version); pe.setEvent(streamEvent); PartitionedEventSerializerImpl peSer = new PartitionedEventSerializerImpl(bolt); byte[] serializedEvent = peSer.serialize(pe); return new TupleImpl(context, Collections.singletonList(serializedEvent), taskId, TEST_STREAM); }
@Test public void testJavaSerialization() { PartitionedEvent partitionedEvent = new PartitionedEvent(); partitionedEvent.setPartitionKey(partitionedEvent.hashCode()); partitionedEvent.setPartition(createSampleStreamGroupbyPartition("sampleStream", Arrays.asList("name", "host"))); StreamEvent event = new StreamEvent(); event.setStreamId("sampleStream"); event.setTimestamp(System.currentTimeMillis()); event.setData(new Object[] {"CPU", "LOCALHOST", true, Long.MAX_VALUE, 60.0}); partitionedEvent.setEvent(event); int javaSerializationLength = SerializationUtils.serialize(partitionedEvent).length; LOG.info("Java serialization length: {}, event: {}", javaSerializationLength, partitionedEvent); int compactLength = 0; compactLength += "sampleStream".getBytes().length; compactLength += ByteUtils.intToBytes(partitionedEvent.getPartition().hashCode()).length; compactLength += ByteUtils.longToBytes(partitionedEvent.getTimestamp()).length; compactLength += "CPU".getBytes().length; compactLength += "LOCALHOST".getBytes().length; compactLength += 1; compactLength += ByteUtils.longToBytes(Long.MAX_VALUE).length; compactLength += ByteUtils.doubleToBytes(60.0).length; LOG.info("Compact serialization length: {}, event: {}", compactLength, partitionedEvent); Assert.assertTrue(compactLength * 20 < javaSerializationLength); }
when(context.getComponentOutputFields("comp1", "default")).thenReturn(new Fields("f0")); PartitionedEvent pe = new PartitionedEvent(); pe.setPartitionKey(1); pe.setPartition(createPartition());
event1.setData(data); event1.setStreamId("correlatedStream"); PartitionedEvent partitionedEvent1 = new PartitionedEvent(event1, sp, 1001); event2.setData(data); event2.setStreamId("correlatedStream"); PartitionedEvent partitionedEvent2 = new PartitionedEvent(event2, sp, 1001);
@Test public void testPartitionedEvent() { PartitionedEvent partitionedEvent = new PartitionedEvent(); Assert.assertEquals("PartitionedEvent[partition=null,event=null,key=0", partitionedEvent.toString()); Object[] data = new Object[]{"namevalue", "hostvalue", "1", 10, 0.1, -0.2, "{\"name\":\"heap.COMMITTED\", \"Value\":\"175636480\"}", 1}; StreamEvent streamEvent = new StreamEvent("streamId", 1478667686971l, data); StreamSortSpec streamSortSpec = new StreamSortSpec(); streamSortSpec.setWindowPeriod("PT10S"); StreamPartition streamPartition = new StreamPartition(); List<String> columns = new ArrayList<>(); columns.add("jobId"); streamPartition.setColumns(columns); streamPartition.setSortSpec(streamSortSpec); streamPartition.setStreamId("test"); streamPartition.setType(StreamPartition.Type.GROUPBY); partitionedEvent = new PartitionedEvent(streamEvent, streamPartition, 1); Assert.assertEquals("PartitionedEvent[partition=StreamPartition[streamId=test,type=GROUPBY,columns=[jobId],sortSpec=[StreamSortSpec[windowPeriod=PT10S,windowMargin=30000]]],event=StreamEvent[stream=STREAMID,timestamp=2016-11-09 05:01:26,971,data=[namevalue,hostvalue,1,10,0.1,-0.2,{\"name\":\"heap.COMMITTED\", \"Value\":\"175636480\"},1],metaVersion=null],key=1", partitionedEvent.toString()); PartitionedEvent partitionedEventCopy = partitionedEvent.copy(); Assert.assertFalse(partitionedEventCopy == partitionedEvent); Assert.assertTrue(partitionedEventCopy.equals(partitionedEvent)); Assert.assertTrue(partitionedEventCopy.hashCode() == partitionedEvent.hashCode()); } }
private PartitionedEvent constructPE() { StreamEvent e = new StreamEvent(); e.setStreamId("testStreamId"); e.setTimestamp(1463159382000L); e.setData(data); StreamPartition sp = new StreamPartition(); List<String> col = new ArrayList<>(); col.add("host"); sp.setColumns(col); StreamSortSpec sortSpec = new StreamSortSpec(); sortSpec.setWindowMargin(30000); sortSpec.setWindowPeriod("PT1M"); sp.setSortSpec(sortSpec); sp.setStreamId("testStreamId"); sp.setType(StreamPartition.Type.GROUPBY); PartitionedEvent pe = new PartitionedEvent(); pe.setEvent(e); pe.setPartition(sp); pe.setPartitionKey(1000); return pe; }
@SuppressWarnings("serial") public static PartitionedEvent createSimpleStreamEvent() { StreamEvent event = StreamEvent.builder() .schema(createSampleStreamDefinition("sampleStream_1")) .streamId("sampleStream_1") .timestamep(System.currentTimeMillis()) .attributes(new HashMap<String, Object>() {{ put("name", "cpu"); put("host", "localhost"); put("flag", true); put("value", 60.0); put("data", Long.MAX_VALUE); put("unknown", "unknown column value"); }}).build(); PartitionedEvent pEvent = new PartitionedEvent(); pEvent.setEvent(event); pEvent.setPartition(createSampleStreamGroupbyPartition("sampleStream_1", Arrays.asList("name", "host"))); return pEvent; } }
public static PartitionedEvent createSimpleStreamEvent() { StreamEvent event = null; try { event = StreamEvent.builder() .schema(MockSampleMetadataFactory.createSingletonMetadataServiceWithSample().getStreamDefinition("sampleStream_1")) .streamId("sampleStream_1") .timestamep(System.currentTimeMillis()) .attributes(new HashMap<String, Object>() {{ put("name", "cpu"); put("value", 60.0); put("unknown", "unknown column value"); }}).build(); } catch (StreamNotDefinedException e) { e.printStackTrace(); } PartitionedEvent pEvent = new PartitionedEvent(); pEvent.setEvent(event); return pEvent; }
PartitionedEvent event1 = new PartitionedEvent(); StreamPartition sp = new StreamPartition(); sp.setColumns(Arrays.asList("host")); PartitionedEvent event2 = new PartitionedEvent(); event2.setPartition(sp); event2.setPartitionKey(1000); PartitionedEvent event3 = new PartitionedEvent(); event3.setPartition(sp); event3.setPartitionKey(1000); event3.setEvent(e3); PartitionedEvent event4 = new PartitionedEvent(); event4.setPartition(sp); event4.setPartitionKey(1000);