private boolean isBoltAvailable(AlertBoltUsage alertUsage) { // FIXME : more detail to compare on queue exclusion check if (alertUsage.getPartitions().stream().filter(partition -> partition.isDedicated()).count() > 0) { return false; } if (!reuseBoltInStreams && alertUsage.getQueueSize() > 0) { return false; } if (reuseBoltInStreams) { if (alertUsage.getQueueSize() >= streamsPerBolt) { return false; } if (alertUsage.getPartitions().contains(partitionGroup)) { return false; } } return alertUsage.getPolicies().size() < numOfPoliciesBoundPerBolt; }
private void addBoltUsageInfo(Map<String, Set<String>> bolt2Policies, Map<String, Set<StreamGroup>> bolt2Partition, Map<String, Set<String>> bolt2QueueIds, String uniqueAlertBolt, AlertBoltUsage alertUsage, Map<String, StreamWorkSlotQueue> queueMap) { // if (bolt2Policies.containsKey(uniqueAlertBolt)) { alertUsage.getPolicies().addAll(bolt2Policies.get(uniqueAlertBolt)); } // if (bolt2Partition.containsKey(uniqueAlertBolt)) { alertUsage.getPartitions().addAll(bolt2Partition.get(uniqueAlertBolt)); } // if (bolt2QueueIds.containsKey(uniqueAlertBolt)) { for (String qId : bolt2QueueIds.get(uniqueAlertBolt)) { if (queueMap.containsKey(qId)) { alertUsage.getReferQueues().add(queueMap.get(qId)); } else { LOG.error(" queue id {} not found in queue map !", qId); } } } }
private void placePolicy(PolicyDefinition def, AlertBoltUsage alertBoltUsage, Topology targetTopology, TopologyUsage usage) { String policyName = def.getName(); // topology usage update alertBoltUsage.addPolicies(def); // update alert policy usage.getPolicies().add(policyName); // update source topics updateDataSource(usage, def); // update group-by updateGrouping(usage, def); }
private boolean isBoltAvailable(AlertBoltUsage boltUsage, PolicyDefinition def) { // overload or over policy # or already contains if (boltUsage == null || boltUsage.getLoad() > boltLoadUpbound || boltUsage.getPolicies().size() >= policiesPerBolt || boltUsage.getPolicies().contains(def.getName())) { return false; } return true; }
@Test public void test() { InMemMetadataServiceClient client = getSampleMetadataService(); ScheduleContextBuilder builder = new ScheduleContextBuilder(config, client); IScheduleContext context = builder.buildContext(); // assert topology usage Map<String, TopologyUsage> usages = context.getTopologyUsages(); Assert.assertEquals(1, usages.get(TOPO1).getMonitoredStream().size()); Assert.assertTrue(usages.get(TOPO1).getPolicies().contains(TEST_POLICY_1)); String alertBolt0 = TOPO1 + "-alert-" + "0"; String alertBolt1 = TOPO1 + "-alert-" + "1"; String alertBolt2 = TOPO1 + "-alert-" + "2"; for (AlertBoltUsage u : usages.get(TOPO1).getAlertUsages().values()) { if (u.getBoltId().equals(alertBolt0) || u.getBoltId().equals(alertBolt1) || u.getBoltId().equals(alertBolt2)) { Assert.assertEquals(1, u.getPolicies().size()); Assert.assertTrue(u.getPolicies().contains(TEST_POLICY_1)); Assert.assertEquals(1, u.getPartitions().size()); Assert.assertEquals(1, u.getReferQueues().size()); } } }
TopologyUsage usage = it.next(); for (AlertBoltUsage u : usage.getAlertUsages().values()) { Assert.assertTrue(u.getPartitions().size() == 0); Assert.assertTrue(u.getQueueSize() == 0); LOG.info(u.getBoltId()); Assert.assertTrue(u.getPartitions().size() == 0); Assert.assertTrue(u.getBoltId(), u.getQueueSize() == 0);
for (String policyName : boltUsage.getPolicies()) { PolicyDefinition definition = context.getPolicies().get(policyName); alertSpec.addBoltPolicy(boltUsage.getBoltId(), definition.getName());
String uniqueBoltId = String.format(UNIQUE_BOLT_ID, t.getName(), alertBolt); AlertBoltUsage alertUsage = new AlertBoltUsage(alertBolt); u.getAlertUsages().put(alertBolt, alertUsage);
private boolean getQueueOnTopology(int size, List<WorkSlot> slots, Topology t) { TopologyUsage u = context.getTopologyUsages().get(t.getName()); if (!isTopologyAvailable(u)) { return false; } List<String> bolts = new ArrayList<String>(); for (AlertBoltUsage alertUsage : u.getAlertUsages().values()) { if (isBoltAvailable(alertUsage)) { bolts.add(alertUsage.getBoltId()); } if (bolts.size() == size) { break; } } if (bolts.size() == size) { for (String boltId : bolts) { WorkSlot slot = new WorkSlot(t.getName(), boltId); slots.add(slot); } return true; } return false; }
Assert.assertEquals(0, topologyUsage.getAlertBoltUsage(slot.getBoltId()).getReferQueues().size());
private void assignQueueSlots(MonitoredStream stream, StreamWorkSlotQueue queue) { for (WorkSlot slot : queue.getWorkingSlots()) { TopologyUsage u = context.getTopologyUsages().get(slot.getTopologyName()); AlertBoltUsage boltUsage = u.getAlertBoltUsage(slot.getBoltId()); boltUsage.addQueue(stream.getStreamGroup(), queue); u.addMonitoredStream(stream); } }
Assert.assertTrue(u.getPartitions().size() > 0); Assert.assertTrue(u.getBoltId(), u.getQueueSize() > 0);
public Pair<Topology, TopologyUsage> createEmptyTopology(String topoName) { Topology t = new Topology(topoName, routerNumber, boltNumber); for (int i = 0; i < t.getNumOfGroupBolt(); i++) { t.getGroupNodeIds().add(t.getName() + "-grp-" + i); } for (int i = 0; i < t.getNumOfAlertBolt(); i++) { t.getAlertBoltIds().add(t.getName() + "-alert-" + i); } TopologyUsage u = new TopologyUsage(topoName); for (String gnid : t.getGroupNodeIds()) { u.getGroupUsages().put(gnid, new GroupBoltUsage(gnid)); } for (String anid : t.getAlertBoltIds()) { u.getAlertUsages().put(anid, new AlertBoltUsage(anid)); } return Pair.of(t, u); } }
u.getAlertUsages().put(anid, new AlertBoltUsage(anid));