public StreamEventBuilder() { instance = new StreamEvent(); }
@Override public StreamEvent deserialize(DataInput dataInput) throws IOException { StreamEvent event = new StreamEvent(); String metaVersionStreamId = dataInput.readUTF(); String streamId = metaVersionStreamId.split("/")[1];
@Override public List<StreamEvent> map(Tuple tuple) throws Exception { long timestamp; if (tuple.getFields().contains(TIMESTAMP_FIELD)) { try { timestamp = tuple.getLongByField("timestamp"); } catch (Exception ex) { // if timestamp is not null LOGGER.error(ex.getMessage(), ex); timestamp = 0; } } else { timestamp = System.currentTimeMillis(); } Object[] values = new Object[tuple.getFields().size()]; for (int i = 0; i < tuple.getFields().size(); i++) { values[i] = tuple.getValue(i); } StreamEvent event = new StreamEvent(); event.setTimestamp(timestamp); event.setStreamId(streamId); event.setData(values); return Collections.singletonList(event); } }
private StreamEvent buildStreamEvt(long ts, String host, double value) { StreamEvent e = new StreamEvent(); e.setData(new Object[] {ts, host, value}); e.setStreamId(inputStream); e.setTimestamp(ts); return e; } }
private StreamEvent buildStreamEvent(long ts, String host, double value) { StreamEvent e = new StreamEvent(); e.setData(new Object[] {ts, host, value}); e.setStreamId(inputStream); e.setTimestamp(ts); return e; }
private StreamEvent buildStreamEvt(long ts, String host, double value) { StreamEvent e = new StreamEvent(); e.setData(new Object[] {ts, host, value}); e.setStreamId(inputStream); e.setTimestamp(ts); return e; }
private StreamEvent buildStreamEvt(long ts, String jobID, String status) { StreamEvent e = new StreamEvent(); e.setData(new Object[] {ts, jobID, status}); e.setStreamId(inputStream); e.setTimestamp(ts); return e; } }
/** * @return cloned new event object. */ public StreamEvent copy() { StreamEvent newEvent = new StreamEvent(); newEvent.setTimestamp(this.getTimestamp()); newEvent.setData(this.getData()); newEvent.setStreamId(this.getStreamId()); newEvent.setMetaVersion(this.getMetaVersion()); return newEvent; }
private Tuple createTuple(AlertBolt bolt, String version) throws IOException { GeneralTopologyContext context = mock(GeneralTopologyContext.class); int taskId = 1; when(context.getComponentId(taskId)).thenReturn("comp1"); when(context.getComponentOutputFields("comp1", TEST_STREAM)).thenReturn(new Fields("f0")); // case 1: bolt prepared but metadata not initialized (no bolt.onAlertBoltSpecChange) PartitionedEvent pe = new PartitionedEvent(); pe.setPartitionKey(1); pe.setPartition(createPartition()); StreamEvent streamEvent = new StreamEvent(); streamEvent.setStreamId(TEST_STREAM); streamEvent.setTimestamp(System.currentTimeMillis()); streamEvent.setMetaVersion(version); pe.setEvent(streamEvent); PartitionedEventSerializerImpl peSer = new PartitionedEventSerializerImpl(bolt); byte[] serializedEvent = peSer.serialize(pe); return new TupleImpl(context, Collections.singletonList(serializedEvent), taskId, TEST_STREAM); }
@Test public void testJavaSerialization() { PartitionedEvent partitionedEvent = new PartitionedEvent(); partitionedEvent.setPartitionKey(partitionedEvent.hashCode()); partitionedEvent.setPartition(createSampleStreamGroupbyPartition("sampleStream", Arrays.asList("name", "host"))); StreamEvent event = new StreamEvent(); event.setStreamId("sampleStream"); event.setTimestamp(System.currentTimeMillis()); event.setData(new Object[] {"CPU", "LOCALHOST", true, Long.MAX_VALUE, 60.0}); partitionedEvent.setEvent(event); int javaSerializationLength = SerializationUtils.serialize(partitionedEvent).length; LOG.info("Java serialization length: {}, event: {}", javaSerializationLength, partitionedEvent); int compactLength = 0; compactLength += "sampleStream".getBytes().length; compactLength += ByteUtils.intToBytes(partitionedEvent.getPartition().hashCode()).length; compactLength += ByteUtils.longToBytes(partitionedEvent.getTimestamp()).length; compactLength += "CPU".getBytes().length; compactLength += "LOCALHOST".getBytes().length; compactLength += 1; compactLength += ByteUtils.longToBytes(Long.MAX_VALUE).length; compactLength += ByteUtils.doubleToBytes(60.0).length; LOG.info("Compact serialization length: {}, event: {}", compactLength, partitionedEvent); Assert.assertTrue(compactLength * 20 < javaSerializationLength); }
pe.setPartitionKey(1); pe.setPartition(createPartition()); StreamEvent streamEvent = new StreamEvent(); streamEvent.setStreamId("test-stream"); streamEvent.setTimestamp(System.currentTimeMillis());
StreamEvent event1 = new StreamEvent(); event1.setTimestamp(DateTimeUtil.humanDateToSeconds("2016-01-01 00:00:00") * 1000); event1.setMetaVersion("version1"); StreamEvent event2 = new StreamEvent(); event2.setTimestamp(DateTimeUtil.humanDateToSeconds("2016-01-01 00:05:00") * 1000); event2.setMetaVersion("version1");
@Test public void testStreamEvent2() { thrown.expect(IndexOutOfBoundsException.class); List<StreamColumn> streamColumns = new ArrayList<>(); streamColumns.add(new StreamColumn.Builder().name("name").type(StreamColumn.Type.STRING).build()); streamColumns.add(new StreamColumn.Builder().name("host").type(StreamColumn.Type.STRING).build()); streamColumns.add(new StreamColumn.Builder().name("flag").type(StreamColumn.Type.BOOL).build()); streamColumns.add(new StreamColumn.Builder().name("value").type(StreamColumn.Type.DOUBLE).build()); streamColumns.add(new StreamColumn.Builder().name("data").type(StreamColumn.Type.LONG).build()); streamColumns.add(new StreamColumn.Builder().name("salary").type(StreamColumn.Type.FLOAT).build()); streamColumns.add(new StreamColumn.Builder().name("object").type(StreamColumn.Type.OBJECT).build()); streamColumns.add(new StreamColumn.Builder().name("int").type(StreamColumn.Type.INT).build()); StreamDefinition streamDefinition = new StreamDefinition(); streamDefinition.setColumns(streamColumns); StreamEvent streamEvent = new StreamEvent(); streamEvent.setData(new Object[]{"namevalue", "hostvalue", "1", 10.0, 1, -0.2, "{\"name\":\"heap.COMMITTED\", \"Value\":\"175636480\"}\""}); streamEvent.getData(streamDefinition, "salary", "int"); }
@Test public void testStreamEvent1() { thrown.expect(IndexOutOfBoundsException.class); List<StreamColumn> streamColumns = new ArrayList<>(); streamColumns.add(new StreamColumn.Builder().name("name").type(StreamColumn.Type.STRING).build()); streamColumns.add(new StreamColumn.Builder().name("host").type(StreamColumn.Type.STRING).build()); streamColumns.add(new StreamColumn.Builder().name("flag").type(StreamColumn.Type.BOOL).build()); streamColumns.add(new StreamColumn.Builder().name("value").type(StreamColumn.Type.DOUBLE).build()); streamColumns.add(new StreamColumn.Builder().name("data").type(StreamColumn.Type.LONG).build()); streamColumns.add(new StreamColumn.Builder().name("salary").type(StreamColumn.Type.FLOAT).build()); streamColumns.add(new StreamColumn.Builder().name("object").type(StreamColumn.Type.OBJECT).build()); streamColumns.add(new StreamColumn.Builder().name("int").type(StreamColumn.Type.INT).build()); StreamDefinition streamDefinition = new StreamDefinition(); streamDefinition.setColumns(streamColumns); StreamEvent streamEvent = new StreamEvent(); streamEvent.setData(new Object[]{"namevalue", "hostvalue", "1", 10.0, 1, -0.2, "{\"name\":\"heap.COMMITTED\", \"Value\":\"175636480\"}\"", 4}); streamEvent.getData(streamDefinition, "salary", "isYhd"); }
@Test public void testStreamEvent3() { List<StreamColumn> streamColumns = new ArrayList<>(); streamColumns.add(new StreamColumn.Builder().name("name").type(StreamColumn.Type.STRING).build()); streamColumns.add(new StreamColumn.Builder().name("host").type(StreamColumn.Type.STRING).build()); streamColumns.add(new StreamColumn.Builder().name("flag").type(StreamColumn.Type.BOOL).build()); streamColumns.add(new StreamColumn.Builder().name("value").type(StreamColumn.Type.DOUBLE).build()); streamColumns.add(new StreamColumn.Builder().name("data").type(StreamColumn.Type.LONG).build()); streamColumns.add(new StreamColumn.Builder().name("salary").type(StreamColumn.Type.FLOAT).build()); streamColumns.add(new StreamColumn.Builder().name("object").type(StreamColumn.Type.OBJECT).build()); streamColumns.add(new StreamColumn.Builder().name("int").type(StreamColumn.Type.INT).build()); StreamDefinition streamDefinition = new StreamDefinition(); streamDefinition.setColumns(streamColumns); StreamEvent streamEvent = new StreamEvent(); streamEvent.setData(new Object[]{"namevalue", 1, "flag", 10.0, 0.1, -0.2, "{\"name\":\"heap.COMMITTED\", \"Value\":\"175636480\"}\"", 1}); Object[] values = streamEvent.getData(streamDefinition, "value", "host"); Assert.assertEquals(10.0, values[0]); Assert.assertEquals(1, values[1]); }
private PartitionedEvent constructPE() { StreamEvent e = new StreamEvent(); e.setStreamId("testStreamId"); e.setTimestamp(1463159382000L); e.setData(data); StreamPartition sp = new StreamPartition(); List<String> col = new ArrayList<>(); col.add("host"); sp.setColumns(col); StreamSortSpec sortSpec = new StreamSortSpec(); sortSpec.setWindowMargin(30000); sortSpec.setWindowPeriod("PT1M"); sp.setSortSpec(sortSpec); sp.setStreamId("testStreamId"); sp.setType(StreamPartition.Type.GROUPBY); PartitionedEvent pe = new PartitionedEvent(); pe.setEvent(e); pe.setPartition(sp); pe.setPartitionKey(1000); return pe; }
StreamEvent event1 = new StreamEvent(); event1.setTimestamp(DateTimeUtil.humanDateToSeconds("2016-01-01 00:00:00") * 1000); event1.setMetaVersion("version1"); StreamEvent event2 = new StreamEvent(); event2.setTimestamp(DateTimeUtil.humanDateToSeconds("2016-01-01 00:00:00") * 1000); event2.setMetaVersion("version1");
@Test public void testPartitionedEvent() { PartitionedEvent partitionedEvent = new PartitionedEvent(); Assert.assertEquals("PartitionedEvent[partition=null,event=null,key=0", partitionedEvent.toString()); Object[] data = new Object[]{"namevalue", "hostvalue", "1", 10, 0.1, -0.2, "{\"name\":\"heap.COMMITTED\", \"Value\":\"175636480\"}", 1}; StreamEvent streamEvent = new StreamEvent("streamId", 1478667686971l, data); StreamSortSpec streamSortSpec = new StreamSortSpec(); streamSortSpec.setWindowPeriod("PT10S"); StreamPartition streamPartition = new StreamPartition(); List<String> columns = new ArrayList<>(); columns.add("jobId"); streamPartition.setColumns(columns); streamPartition.setSortSpec(streamSortSpec); streamPartition.setStreamId("test"); streamPartition.setType(StreamPartition.Type.GROUPBY); partitionedEvent = new PartitionedEvent(streamEvent, streamPartition, 1); Assert.assertEquals("PartitionedEvent[partition=StreamPartition[streamId=test,type=GROUPBY,columns=[jobId],sortSpec=[StreamSortSpec[windowPeriod=PT10S,windowMargin=30000]]],event=StreamEvent[stream=STREAMID,timestamp=2016-11-09 05:01:26,971,data=[namevalue,hostvalue,1,10,0.1,-0.2,{\"name\":\"heap.COMMITTED\", \"Value\":\"175636480\"},1],metaVersion=null],key=1", partitionedEvent.toString()); PartitionedEvent partitionedEventCopy = partitionedEvent.copy(); Assert.assertFalse(partitionedEventCopy == partitionedEvent); Assert.assertTrue(partitionedEventCopy.equals(partitionedEvent)); Assert.assertTrue(partitionedEventCopy.hashCode() == partitionedEvent.hashCode()); } }
@Test public void testStreamEvent() { Object[] data = new Object[]{"namevalue", "hostvalue", "1", 10.0, 1, -0.2, "{\"name\":\"heap.COMMITTED\", \"Value\":\"175636480\"}", 4}; StreamEvent streamEvent = new StreamEvent("streamId", 1478667686971l, data); streamEvent = new StreamEvent("streamId", 1478667686971l, data, "metaVersion");
event1.setPartition(sp); event1.setPartitionKey(1000); StreamEvent e1 = new StreamEvent(); e1.setData(new Object[] {18.4}); e1.setStreamId("testStreamId"); event2.setPartition(sp); event2.setPartitionKey(1000); StreamEvent e2 = new StreamEvent(); e2.setData(new Object[] {16.3}); e2.setStreamId("testStreamId"); event3.setPartition(sp); event3.setPartitionKey(1000); StreamEvent e3 = new StreamEvent(); e3.setData(new Object[] {14.3}); e3.setStreamId("testStreamId"); event4.setPartition(sp); event4.setPartitionKey(1000); StreamEvent e4 = new StreamEvent(); e4.setData(new Object[] {14.3}); e4.setStreamId("testStreamId");