@Test public void testWorkSlot() { WorkSlot workSlot = new WorkSlot(); Assert.assertEquals("(null:null)", workSlot.toString()); Assert.assertEquals(null, workSlot.getBoltId()); Assert.assertEquals(null, workSlot.getTopologyName()); workSlot.setBoltId("setBoltId"); workSlot.setTopologyName("setTopologyName"); Assert.assertEquals("(setTopologyName:setBoltId)", workSlot.toString()); Assert.assertEquals("setBoltId", workSlot.getBoltId()); Assert.assertEquals("setTopologyName", workSlot.getTopologyName()); WorkSlot workSlot1 = new WorkSlot("setTopologyName", "setBoltId"); Assert.assertEquals("(setTopologyName:setBoltId)", workSlot1.toString()); Assert.assertEquals("setBoltId", workSlot1.getBoltId()); Assert.assertEquals("setTopologyName", workSlot1.getTopologyName()); Assert.assertTrue(workSlot1.equals(workSlot)); Assert.assertTrue(workSlot1.hashCode() == workSlot.hashCode()); } }
private String getUniqueBoltId(WorkSlot slot) { return String.format(UNIQUE_BOLT_ID, slot.getTopologyName(), slot.getBoltId()); }
private boolean getQueueOnTopology(int size, List<WorkSlot> slots, Topology t) { TopologyUsage u = context.getTopologyUsages().get(t.getName()); if (!isTopologyAvailable(u)) { return false; } List<String> bolts = new ArrayList<String>(); for (AlertBoltUsage alertUsage : u.getAlertUsages().values()) { if (isBoltAvailable(alertUsage)) { bolts.add(alertUsage.getBoltId()); } if (bolts.size() == size) { break; } } if (bolts.size() == size) { for (String boltId : bolts) { WorkSlot slot = new WorkSlot(t.getName(), boltId); slots.add(slot); } return true; } return false; }
/** * in correlation cases, multiple streams will go to the same queue for correlation policy. * * @param spec */ private void sanityCheck(RouterSpec spec) { Set<String> totalRequestedSlots = new HashSet<>(); for (StreamRouterSpec s : spec.getRouterSpecs()) { for (PolicyWorkerQueue q : s.getTargetQueue()) { List<String> workers = new ArrayList<>(); q.getWorkers().forEach(w -> workers.add(w.getBoltId())); totalRequestedSlots.addAll(workers); } } if (totalRequestedSlots.size() > getOutputStreamIds().size()) { String error = String.format("Requested slots are not consistent with provided slots, %s, %s", totalRequestedSlots, getOutputStreamIds()); LOG.error(error); throw new IllegalStateException(error); } }
private void calculateGroupIndexAndCount(StreamWorkSlotQueue queue) { Map<String, Integer> result = new HashMap<String, Integer>(); int total = 0; for (WorkSlot slot : queue.getWorkingSlots()) { if (result.containsKey(slot.getTopologyName())) { continue; } result.put(slot.getTopologyName(), total); total += context.getTopologies().get(slot.getTopologyName()).getNumOfGroupBolt(); } queue.setNumberOfGroupBolts(total); queue.setTopoGroupStartIndex(result); }
private void placePolicyToQueue(PolicyDefinition def, StreamWorkSlotQueue queue, Map<String, PolicyAssignment> newAssignments) { for (WorkSlot slot : queue.getWorkingSlots()) { Topology targetTopology = context.getTopologies().get(slot.getTopologyName()); TopologyUsage usage = context.getTopologyUsages().get(slot.getTopologyName()); AlertBoltUsage alertBoltUsage = usage.getAlertBoltUsage(slot.getBoltId()); placePolicy(def, alertBoltUsage, targetTopology, usage); } // queue.placePolicy(def); PolicyAssignment assignment = new PolicyAssignment(def.getName(), queue.getQueueId()); context.getPolicyAssignments().put(def.getName(), assignment); newAssignments.put(def.getName(), assignment); }
private static MonitoredStream createMonitoredStream() { MonitoredStream ms = new MonitoredStream(streamGroup); ms.setVersion(V1); List<WorkSlot> slots = new ArrayList<WorkSlot>(); WorkSlot slot0 = new WorkSlot(TOPO1, TOPO1 + "-alert-" + 0); WorkSlot slot1 = new WorkSlot(TOPO1, TOPO1 + "-alert-" + 1); WorkSlot slot2 = new WorkSlot(TOPO1, TOPO1 + "-alert-" + 2); WorkSlot slot3 = new WorkSlot(TOPO1, TOPO1 + "-alert-" + 3); WorkSlot slot4 = new WorkSlot(TOPO1, TOPO1 + "-alert-" + 4); WorkSlot slot5 = new WorkSlot(TOPO1, TOPO1 + "-alert-" + 5); slots.add(slot0); slots.add(slot1); slots.add(slot2); slots.add(slot3); slots.add(slot4); //slots.add(slot5); StreamWorkSlotQueue q = new StreamWorkSlotQueue(streamGroup, false, new HashMap<>(), slots); ms.addQueues(q); queueId = q.getQueueId(); return ms; }
Assert.assertEquals(0, topologyUsage.getAlertBoltUsage(slot.getBoltId()).getReferQueues().size());
if (!topo2MonitorStream.containsKey(slot.getTopologyName())) { topo2MonitorStream.put(slot.getTopologyName(), new HashSet<MonitoredStream>()); topo2MonitorStream.get(slot.getTopologyName()).add(stream); if (!topo2Policies.containsKey(slot.getTopologyName())) { topo2Policies.put(slot.getTopologyName(), new HashSet<String>()); topo2Policies.get(slot.getTopologyName()).addAll(policiesOnQ);
private void assignQueueSlots(MonitoredStream stream, StreamWorkSlotQueue queue) { for (WorkSlot slot : queue.getWorkingSlots()) { TopologyUsage u = context.getTopologyUsages().get(slot.getTopologyName()); AlertBoltUsage boltUsage = u.getAlertBoltUsage(slot.getBoltId()); boltUsage.addQueue(stream.getStreamGroup(), queue); u.addMonitoredStream(stream); } }
public static StreamRouterSpec createSampleStreamRouteSpec(String streamId, String groupByField, List<String> targetEvaluatorIds) { List<WorkSlot> slots = Arrays.asList(targetEvaluatorIds.stream().map((t) -> { return new WorkSlot("sampleTopology", t); }).toArray(WorkSlot[]::new)); StreamRouterSpec streamRouteSpec = new StreamRouterSpec(); streamRouteSpec.setStreamId(streamId); streamRouteSpec.setPartition(createSampleStreamGroupbyPartition(streamId, Arrays.asList(groupByField))); streamRouteSpec.setTargetQueue(Arrays.asList(new PolicyWorkerQueue(slots))); return streamRouteSpec; }
group1Slots.add(slot.getBoltId()); }); Assert.assertEquals(2, getMonitorStream(usage.getMonitoredStream()).get(group2).getQueues().get(0).getWorkingSlots().size()); getMonitorStream(usage.getMonitoredStream()).get(group2).getQueues().get(0).getWorkingSlots().forEach(slot -> { Assert.assertTrue(!group1Slots.contains(slot.getBoltId())); }); Assert.assertEquals(2, getMonitorStream(usage.getMonitoredStream()).get(group3).getQueues().get(0).getWorkingSlots().size()); getMonitorStream(usage.getMonitoredStream()).get(group3).getQueues().get(0).getWorkingSlots().forEach(slot -> { Assert.assertTrue(!group1Slots.contains(slot.getBoltId())); });
List<WorkSlot> slots = strategy.reserveWorkSlots(5, false, new HashMap<String, Object>()); Assert.assertEquals(5, slots.size()); LOG.info(slots.get(0).getTopologyName()); Assert.assertEquals(2, context.getTopologies().size()); Assert.assertEquals(2, context.getTopologyUsages().size()); Assert.assertEquals(slots.get(0).getTopologyName(), ws.getTopologyName());
private boolean isQueueAvailable(StreamWorkSlotQueue queue, PolicyDefinition def) { if (queue.getQueueSize() < def.getParallelismHint()) { return false; } for (WorkSlot slot : queue.getWorkingSlots()) { TopologyUsage u = context.getTopologyUsages().get(slot.getTopologyName()); AlertBoltUsage usage = u.getAlertBoltUsage(slot.getBoltId()); if (!isBoltAvailable(usage, def)) { return false; } } return true; }
streamPartition.setType(StreamPartition.Type.GROUPBY); streamGroup.addStreamPartition(streamPartition); WorkSlot workSlot = new WorkSlot("setTopologyName", "setBoltId"); List<WorkSlot> workSlots = new ArrayList<>(); workSlots.add(workSlot);
group1Slots.add(slot.getBoltId()); }); Assert.assertEquals(2, getMonitorStream(usage.getMonitoredStream()).get(group2).getQueues().get(0).getWorkingSlots().size()); getMonitorStream(usage.getMonitoredStream()).get(group2).getQueues().get(0).getWorkingSlots().forEach(slot -> { Assert.assertTrue(!group1Slots.contains(slot.getBoltId())); });
StreamWorkSlotQueue queue1 = getQueue(context, pa1.getQueueId()).getRight(); StreamWorkSlotQueue queue3 = getQueue(context, pa3.getQueueId()).getRight(); Assert.assertNotEquals(queue1.getWorkingSlots().get(0).getTopologyName(), queue3.getWorkingSlots().get(0).getTopologyName());
topo1 = queue.getWorkingSlots().get(0).getTopologyName(); bolt1 = queue.getWorkingSlots().get(0).getBoltId(); Assert.assertEquals(1, context.getTopologies().size()); Assert.assertEquals(1, context.getTopologyUsages().size()); LOG.info(queue.getWorkingSlots().get(0).getTopologyName()); for (WorkSlot ws : queue.getWorkingSlots()) { Assert.assertEquals(topo1, ws.getTopologyName()); Assert.assertEquals(2, context.getTopologyUsages().size()); String topo2 = queue.getWorkingSlots().get(0).getTopologyName(); String bolt2 = queue.getWorkingSlots().get(0).getBoltId(); for (WorkSlot ws : queue.getWorkingSlots()) { Assert.assertEquals(topo2, ws.getTopologyName());
@Test public void testRouterBoltSpec() { List<String> topics = Arrays.asList("testTopic3", "testTopic4", "testTopic5"); RouterSpec boltSpec = new RouterSpec(); for (String topic : topics) { String streamId = getStreamNameByTopic(topic); // StreamPartition, groupby col1 for stream cpuUsageStream StreamPartition sp = new StreamPartition(); sp.setStreamId(streamId); sp.setColumns(Arrays.asList("value")); sp.setType(StreamPartition.Type.GROUPBY); StreamSortSpec sortSpec = new StreamSortSpec(); sortSpec.setWindowMargin(1000); sortSpec.setWindowPeriod2(Period.seconds(10)); sp.setSortSpec(sortSpec); // set StreamRouterSpec to have 2 WorkSlot StreamRouterSpec routerSpec = new StreamRouterSpec(); routerSpec.setPartition(sp); routerSpec.setStreamId(streamId); PolicyWorkerQueue queue = new PolicyWorkerQueue(); queue.setPartition(sp); queue.setWorkers(Arrays.asList(new WorkSlot("testTopology", "alertBolt0"), new WorkSlot("testTopology", "alertBolt1"))); routerSpec.setTargetQueue(Arrays.asList(queue)); boltSpec.addRouterSpec(routerSpec); } String json = MetadataSerDeser.serialize(boltSpec); System.out.println(json); RouterSpec deserializedSpec = MetadataSerDeser.deserialize(json, RouterSpec.class); Assert.assertEquals(3, deserializedSpec.getRouterSpecs().size()); }
Assert.assertEquals(5, queues.get(0).getWorkers().size()); for (WorkSlot slot : queues.get(0).getWorkers()) { Assert.assertEquals(topo1, slot.getTopologyName()); LOG.info(slot.getBoltId());