TopologyUsage u = new TopologyUsage(t.getName()); u.getGroupUsages().put(grpBolt, grpUsage); u.getAlertUsages().put(alertBolt, alertUsage); if (topo2Policies.containsKey(u.getTopoName())) { u.getPolicies().addAll(topo2Policies.get(u.getTopoName())); if (topo2MonitorStream.containsKey(u.getTopoName())) { u.getMonitoredStream().addAll(topo2MonitorStream.get(u.getTopoName())); usages.put(u.getTopoName(), u);
private void assignQueueSlots(MonitoredStream stream, StreamWorkSlotQueue queue) { for (WorkSlot slot : queue.getWorkingSlots()) { TopologyUsage u = context.getTopologyUsages().get(slot.getTopologyName()); AlertBoltUsage boltUsage = u.getAlertBoltUsage(slot.getBoltId()); boltUsage.addQueue(stream.getStreamGroup(), queue); u.addMonitoredStream(stream); } }
private void buildTopologyDataSource(TopologyUsage u) { for (String policyName : u.getPolicies()) { PolicyDefinition def = policies.get(policyName); if (def != null) { u.getDataSources().addAll(findDatasource(def)); } else { LOG.error(" policy not find {}, but reference in topology usage {} !", policyName, u.getTopoName()); } } }
PublishSpec pubSpec = pubSpecs.get(u.getTopoName()); if (pubSpec == null) { pubSpec = new PublishSpec(u.getTopoName(), context.getTopologies().get(u.getTopoName()).getPubBoltId()); pubSpecs.put(u.getTopoName(), pubSpec); for (String p : u.getPolicies()) { PolicyDefinition definition = context.getPolicies().get(p); if (definition == null) {
public Pair<Topology, TopologyUsage> createEmptyTopology(String topoName) { Topology t = new Topology(topoName, routerNumber, boltNumber); for (int i = 0; i < t.getNumOfGroupBolt(); i++) { t.getGroupNodeIds().add(t.getName() + "-grp-" + i); } for (int i = 0; i < t.getNumOfAlertBolt(); i++) { t.getAlertBoltIds().add(t.getName() + "-alert-" + i); } TopologyUsage u = new TopologyUsage(topoName); for (String gnid : t.getGroupNodeIds()) { u.getGroupUsages().put(gnid, new GroupBoltUsage(gnid)); } for (String anid : t.getAlertBoltIds()) { u.getAlertUsages().put(anid, new AlertBoltUsage(anid)); } return Pair.of(t, u); } }
@Test public void test() { InMemMetadataServiceClient client = getSampleMetadataService(); ScheduleContextBuilder builder = new ScheduleContextBuilder(config, client); IScheduleContext context = builder.buildContext(); // assert topology usage Map<String, TopologyUsage> usages = context.getTopologyUsages(); Assert.assertEquals(1, usages.get(TOPO1).getMonitoredStream().size()); Assert.assertTrue(usages.get(TOPO1).getPolicies().contains(TEST_POLICY_1)); String alertBolt0 = TOPO1 + "-alert-" + "0"; String alertBolt1 = TOPO1 + "-alert-" + "1"; String alertBolt2 = TOPO1 + "-alert-" + "2"; for (AlertBoltUsage u : usages.get(TOPO1).getAlertUsages().values()) { if (u.getBoltId().equals(alertBolt0) || u.getBoltId().equals(alertBolt1) || u.getBoltId().equals(alertBolt2)) { Assert.assertEquals(1, u.getPolicies().size()); Assert.assertTrue(u.getPolicies().contains(TEST_POLICY_1)); Assert.assertEquals(1, u.getPartitions().size()); Assert.assertEquals(1, u.getReferQueues().size()); } } }
Set<String> topoPolicies = topologyUsage.getPolicies(); Assert.assertFalse(topoPolicies.contains(TEST_DATASOURCE_1)); Assert.assertEquals(0, topoPolicies.size()); Assert.assertEquals(0, topologyUsage.getMonitoredStream().size()); Assert.assertEquals(0, topologyUsage.getAlertBoltUsage(slot.getBoltId()).getReferQueues().size());
private boolean getQueueOnTopology(int size, List<WorkSlot> slots, Topology t) { TopologyUsage u = context.getTopologyUsages().get(t.getName()); if (!isTopologyAvailable(u)) { return false; } List<String> bolts = new ArrayList<String>(); for (AlertBoltUsage alertUsage : u.getAlertUsages().values()) { if (isBoltAvailable(alertUsage)) { bolts.add(alertUsage.getBoltId()); } if (bolts.size() == size) { break; } } if (bolts.size() == size) { for (String boltId : bolts) { WorkSlot slot = new WorkSlot(t.getName(), boltId); slots.add(slot); } return true; } return false; }
private Map<String, AlertBoltSpec> generateAlertMonitorMetadata() { Map<String, AlertBoltSpec> alertSpecs = new HashMap<String, AlertBoltSpec>(); for (TopologyUsage u : context.getTopologyUsages().values()) { AlertBoltSpec alertSpec = alertSpecs.get(u.getTopoName()); if (alertSpec == null) { alertSpec = new AlertBoltSpec(u.getTopoName()); alertSpecs.put(u.getTopoName(), alertSpec); for (AlertBoltUsage boltUsage : u.getAlertUsages().values()) { for (String policyName : boltUsage.getPolicies()) { PolicyDefinition definition = context.getPolicies().get(policyName);
private Map<String, RouterSpec> generateGroupbyMonitorMetadata() { Map<String, RouterSpec> groupSpecsMap = new HashMap<String, RouterSpec>(); for (TopologyUsage u : context.getTopologyUsages().values()) { RouterSpec spec = groupSpecsMap.get(u.getTopoName()); if (spec == null) { spec = new RouterSpec(u.getTopoName()); groupSpecsMap.put(u.getTopoName(), spec); } for (MonitoredStream ms : u.getMonitoredStream()) { // mutiple stream on the same policy group : for correlation group case: for (StreamPartition partiton : ms.getStreamGroup().getStreamPartitions()) { StreamRouterSpec routeSpec = new StreamRouterSpec(); routeSpec.setPartition(partiton); routeSpec.setStreamId(partiton.getStreamId()); for (StreamWorkSlotQueue sq : ms.getQueues()) { if (sq.getTopoGroupStartIndex().containsKey(u.getTopoName())) { PolicyWorkerQueue queue = new PolicyWorkerQueue(); queue.setWorkers(sq.getWorkingSlots()); queue.setPartition(partiton); routeSpec.addQueue(queue); } } spec.addRouterSpec(routeSpec); } } } return groupSpecsMap; }
private void placePolicy(PolicyDefinition def, AlertBoltUsage alertBoltUsage, Topology targetTopology, TopologyUsage usage) { String policyName = def.getName(); // topology usage update alertBoltUsage.addPolicies(def); // update alert policy usage.getPolicies().add(policyName); // update source topics updateDataSource(usage, def); // update group-by updateGrouping(usage, def); }
private boolean isQueueAvailable(StreamWorkSlotQueue queue, PolicyDefinition def) { if (queue.getQueueSize() < def.getParallelismHint()) { return false; } for (WorkSlot slot : queue.getWorkingSlots()) { TopologyUsage u = context.getTopologyUsages().get(slot.getTopologyName()); AlertBoltUsage usage = u.getAlertBoltUsage(slot.getBoltId()); if (!isBoltAvailable(usage, def)) { return false; } } return true; }
Assert.assertTrue(getMonitorStream(usage.getMonitoredStream()).containsKey(group1)); Assert.assertEquals(1, getMonitorStream(usage.getMonitoredStream()).get(group1).getQueues().size()); Assert.assertEquals(2, getMonitorStream(usage.getMonitoredStream()).get(group1).getQueues().get(0).getWorkingSlots().size()); getMonitorStream(usage.getMonitoredStream()).get(group1).getQueues().get(0).getWorkingSlots().forEach(slot -> { group1Slots.add(slot.getBoltId()); }); print(context.getTopologyUsages().values()); Assert.assertTrue(getMonitorStream(usage.getMonitoredStream()).containsKey(group2)); Assert.assertEquals(1, getMonitorStream(usage.getMonitoredStream()).get(group2).getQueues().size()); Assert.assertEquals(2, getMonitorStream(usage.getMonitoredStream()).get(group2).getQueues().get(0).getWorkingSlots().size()); getMonitorStream(usage.getMonitoredStream()).get(group2).getQueues().get(0).getWorkingSlots().forEach(slot -> { Assert.assertTrue(!group1Slots.contains(slot.getBoltId())); }); print(context.getTopologyUsages().values()); Assert.assertTrue(getMonitorStream(usage.getMonitoredStream()).containsKey(group3)); Assert.assertEquals(1, getMonitorStream(usage.getMonitoredStream()).get(group3).getQueues().size()); Assert.assertEquals(2, getMonitorStream(usage.getMonitoredStream()).get(group3).getQueues().get(0).getWorkingSlots().size()); getMonitorStream(usage.getMonitoredStream()).get(group3).getQueues().get(0).getWorkingSlots().forEach(slot -> { Assert.assertTrue(!group1Slots.contains(slot.getBoltId())); }); Assert.assertTrue(!getMonitorStream(usage.getMonitoredStream()).containsKey(group4));
private boolean isTopologyAvailable(TopologyUsage u) { // for (MonitoredStream stream : u.getMonitoredStream()) { // if (partition.equals(stream.getStreamParitition())) { // return false; // } // } if (u == null || u.getLoad() > topoLoadUpbound) { return false; } return true; }
public void addTopologyUsages(TopologyUsage usage) { usages.put(usage.getTopoName(), usage); }
private void updateDataSource(TopologyUsage usage, PolicyDefinition def) { List<String> datasources = findDatasource(def); usage.getDataSources().addAll(datasources); }
context.addTopology(t); TopologyUsage u = new TopologyUsage(t.getName()); for (String gnid : t.getGroupNodeIds()) { u.getGroupUsages().put(gnid, new GroupBoltUsage(gnid)); u.getAlertUsages().put(anid, new AlertBoltUsage(anid));
for (AlertBoltUsage u : usage.getAlertUsages().values()) { Assert.assertTrue(u.getPartitions().size() == 0); Assert.assertTrue(u.getQueueSize() == 0); for (AlertBoltUsage u : usage.getAlertUsages().values()) { LOG.info(u.getBoltId()); Assert.assertTrue(u.getPartitions().size() == 0);
@Test public void test_remove_policy() { InMemMetadataServiceClient client = getSampleMetadataService(); ScheduleContextBuilder builder = new ScheduleContextBuilder(config, client); PolicyAssignment assignment1 = client.getVersionedSpec().getAssignments().get(0); IScheduleContext context = builder.buildContext(); Assert.assertTrue(context.getPolicyAssignments().containsKey(TEST_POLICY_1)); StreamWorkSlotQueue queue = SchedulerTest.getQueue(context, assignment1.getQueueId()).getRight(); client.removePolicy(0); context = builder.buildContext(); Assert.assertFalse(context.getPolicyAssignments().containsKey(TEST_POLICY_1)); WorkSlot slot = queue.getWorkingSlots().get(0); Set<String> topoPolicies = context.getTopologyUsages().get(slot.topologyName).getPolicies(); Assert.assertFalse(topoPolicies.contains(TEST_DATASOURCE_1)); Assert.assertEquals(0, topoPolicies.size()); }
private void placePolicyToQueue(PolicyDefinition def, StreamWorkSlotQueue queue, Map<String, PolicyAssignment> newAssignments) { for (WorkSlot slot : queue.getWorkingSlots()) { Topology targetTopology = context.getTopologies().get(slot.getTopologyName()); TopologyUsage usage = context.getTopologyUsages().get(slot.getTopologyName()); AlertBoltUsage alertBoltUsage = usage.getAlertBoltUsage(slot.getBoltId()); placePolicy(def, alertBoltUsage, targetTopology, usage); } // queue.placePolicy(def); PolicyAssignment assignment = new PolicyAssignment(def.getName(), queue.getQueueId()); context.getPolicyAssignments().put(def.getName(), assignment); newAssignments.put(def.getName(), assignment); }