public boolean isPartitionRequired() { return this.getPartition() != null; }
@Override public boolean equals(Object obj) { if (obj == this) { return true; } if (obj == null) { return false; } if (obj instanceof PartitionedEvent) { PartitionedEvent another = (PartitionedEvent) obj; return !(this.partitionKey != another.getPartitionKey() || !Objects.equals(this.event, another.getEvent()) || !Objects.equals(this.partition, another.getPartition()) || !Objects.equals(this.anchor, another.anchor)); } else { return false; } }
@Override public void serialize(PartitionedEvent entity, DataOutput dataOutput) throws IOException { dataOutput.writeLong(entity.getPartitionKey()); streamEventSerializer.serialize(entity.getEvent(), dataOutput); streamPartitionSerializer.serialize(entity.getPartition(), dataOutput); }
public boolean isSortRequired() { return isPartitionRequired() && this.getPartition().getSortSpec() != null; }
@Override public void serialize(PartitionedEvent entity, DataOutput dataOutput) throws IOException { dataOutput.writeLong(entity.getPartitionKey()); streamEventSerializer.serialize(entity.getEvent(), dataOutput); streamPartitionSerializer.serialize(entity.getPartition(), dataOutput); }
private static boolean isAcceptedByPolicy(PartitionedEvent event, PolicyDefinition policy) { return policy.getPartitionSpec().contains(event.getPartition()) && (policy.getInputStreams().contains(event.getEvent().getStreamId()) || policy.getDefinition().getInputStreams().contains(event.getEvent().getStreamId())); }
@Override public List<Integer> chooseTasks(int taskId, List<Object> values) { Object routingKeyObj = values.get(0); if (routingKeyObj != null) { PartitionedEvent partitionedEvent = (PartitionedEvent) routingKeyObj; if (partitionedEvent.getPartition().getType() == StreamPartition.Type.GLOBAL) { return globalGroupingDelegate.chooseTasks(taskId, values); } else if (partitionedEvent.getPartition().getType() == StreamPartition.Type.GROUPBY) { return Collections.singletonList(outdegreeTasks.get((int) (partitionedEvent.getPartitionKey() % this.outdegreeTasks.size()))); } // Shuffle by defaults return shuffleGroupingDelegate.chooseTasks(taskId, values); } LOG.warn("Illegal null StreamRoute, throw event"); return Collections.emptyList(); } }
/** * @param event input event. * @return whether sorted. */ private boolean dispatchToSortHandler(PartitionedEvent event) { if (event.getTimestamp() <= 0) { return false; } StreamSortHandler sortHandler = streamSortHandlers.get(event.getPartition()); if (sortHandler == null) { if (event.isSortRequired()) { LOG.warn("Stream sort handler required has not been loaded so emmit directly: {}", event); this.context.counter().incr("miss_sort_count"); } return false; } else { sortHandler.nextEvent(event); return true; } }
private void writePartitionedEvent(DataOutput2 out, PartitionedEvent event) throws IOException { out.packLong(event.getPartitionKey()); int partitionHashCode = 0; if (event.getPartition() != null) { partitionHashCode = event.getPartition().hashCode(); if (!hashCodePartitionDict.containsKey(partitionHashCode)) { hashCodePartitionDict.put(partitionHashCode, event.getPartition()); } } out.packInt(partitionHashCode); if (event.getEvent() != null) { byte[] eventBytes = SerializableUtils.serializeToCompressedByteArray(event.getEvent()); out.packInt(eventBytes.length); out.write(eventBytes); } else { out.packInt(0); } }
public void emit(PartitionedEvent event) { try { this.streamContext.counter().incr("send_count"); StreamPartition partition = event.getPartition(); List<StreamRouterSpec> routerSpecs = routeSpecMap.get(partition); if (routerSpecs == null || routerSpecs.size() <= 0) { if (LOG.isDebugEnabled()) { LOG.debug("Drop event {} as StreamPartition {} is not pointed to any router metadata {}", event, event.getPartition(), routeSpecMap);
@Test public void testJavaSerialization() { PartitionedEvent partitionedEvent = new PartitionedEvent(); partitionedEvent.setPartitionKey(partitionedEvent.hashCode()); partitionedEvent.setPartition(createSampleStreamGroupbyPartition("sampleStream", Arrays.asList("name", "host"))); StreamEvent event = new StreamEvent(); event.setStreamId("sampleStream"); event.setTimestamp(System.currentTimeMillis()); event.setData(new Object[] {"CPU", "LOCALHOST", true, Long.MAX_VALUE, 60.0}); partitionedEvent.setEvent(event); int javaSerializationLength = SerializationUtils.serialize(partitionedEvent).length; LOG.info("Java serialization length: {}, event: {}", javaSerializationLength, partitionedEvent); int compactLength = 0; compactLength += "sampleStream".getBytes().length; compactLength += ByteUtils.intToBytes(partitionedEvent.getPartition().hashCode()).length; compactLength += ByteUtils.longToBytes(partitionedEvent.getTimestamp()).length; compactLength += "CPU".getBytes().length; compactLength += "LOCALHOST".getBytes().length; compactLength += 1; compactLength += ByteUtils.longToBytes(Long.MAX_VALUE).length; compactLength += ByteUtils.doubleToBytes(60.0).length; LOG.info("Compact serialization length: {}, event: {}", compactLength, partitionedEvent); Assert.assertTrue(compactLength * 20 < javaSerializationLength); }