public InMemScheduleConext(IScheduleContext context) { this.topologies = new HashMap<String, Topology>(context.getTopologies()); this.usages = new HashMap<String, TopologyUsage>(context.getTopologyUsages()); this.policies = new HashMap<String, PolicyDefinition>(context.getPolicies()); this.datasources = new HashMap<String, Kafka2TupleMetadata>(context.getDataSourceMetadata()); this.policyAssignments = new HashMap<String, PolicyAssignment>(context.getPolicyAssignments()); this.schemas = new HashMap<String, StreamDefinition>(context.getStreamSchemas()); this.monitoredStreams = new HashMap<StreamGroup, MonitoredStream>(context.getMonitoredStreams()); this.publishments = new HashMap<String, Publishment>(context.getPublishments()); }
context.getTopologyUsages().put(topoMeta.topologyId, topoMeta.usage); boolean placed = getQueueOnTopology(size, slots, topoMeta.topology); if (!placed) {
private boolean getQueueOnTopology(int size, List<WorkSlot> slots, Topology t) { TopologyUsage u = context.getTopologyUsages().get(t.getName()); if (!isTopologyAvailable(u)) { return false; } List<String> bolts = new ArrayList<String>(); for (AlertBoltUsage alertUsage : u.getAlertUsages().values()) { if (isBoltAvailable(alertUsage)) { bolts.add(alertUsage.getBoltId()); } if (bolts.size() == size) { break; } } if (bolts.size() == size) { for (String boltId : bolts) { WorkSlot slot = new WorkSlot(t.getName(), boltId); slots.add(slot); } return true; } return false; }
for (TopologyUsage u : context.getTopologyUsages().values()) { PublishSpec pubSpec = pubSpecs.get(u.getTopoName()); if (pubSpec == null) {
private void placePolicyToQueue(PolicyDefinition def, StreamWorkSlotQueue queue, Map<String, PolicyAssignment> newAssignments) { for (WorkSlot slot : queue.getWorkingSlots()) { Topology targetTopology = context.getTopologies().get(slot.getTopologyName()); TopologyUsage usage = context.getTopologyUsages().get(slot.getTopologyName()); AlertBoltUsage alertBoltUsage = usage.getAlertBoltUsage(slot.getBoltId()); placePolicy(def, alertBoltUsage, targetTopology, usage); } // queue.placePolicy(def); PolicyAssignment assignment = new PolicyAssignment(def.getName(), queue.getQueueId()); context.getPolicyAssignments().put(def.getName(), assignment); newAssignments.put(def.getName(), assignment); }
private void assignQueueSlots(MonitoredStream stream, StreamWorkSlotQueue queue) { for (WorkSlot slot : queue.getWorkingSlots()) { TopologyUsage u = context.getTopologyUsages().get(slot.getTopologyName()); AlertBoltUsage boltUsage = u.getAlertBoltUsage(slot.getBoltId()); boltUsage.addQueue(stream.getStreamGroup(), queue); u.addMonitoredStream(stream); } }
private boolean isQueueAvailable(StreamWorkSlotQueue queue, PolicyDefinition def) { if (queue.getQueueSize() < def.getParallelismHint()) { return false; } for (WorkSlot slot : queue.getWorkingSlots()) { TopologyUsage u = context.getTopologyUsages().get(slot.getTopologyName()); AlertBoltUsage usage = u.getAlertBoltUsage(slot.getBoltId()); if (!isBoltAvailable(usage, def)) { return false; } } return true; }
private Map<String, AlertBoltSpec> generateAlertMonitorMetadata() { Map<String, AlertBoltSpec> alertSpecs = new HashMap<String, AlertBoltSpec>(); for (TopologyUsage u : context.getTopologyUsages().values()) { AlertBoltSpec alertSpec = alertSpecs.get(u.getTopoName()); if (alertSpec == null) {
for (TopologyUsage usage : context.getTopologyUsages().values()) { Topology topo = context.getTopologies().get(usage.getTopoName());
private Map<String, RouterSpec> generateGroupbyMonitorMetadata() { Map<String, RouterSpec> groupSpecsMap = new HashMap<String, RouterSpec>(); for (TopologyUsage u : context.getTopologyUsages().values()) { RouterSpec spec = groupSpecsMap.get(u.getTopoName()); if (spec == null) { spec = new RouterSpec(u.getTopoName()); groupSpecsMap.put(u.getTopoName(), spec); } for (MonitoredStream ms : u.getMonitoredStream()) { // mutiple stream on the same policy group : for correlation group case: for (StreamPartition partiton : ms.getStreamGroup().getStreamPartitions()) { StreamRouterSpec routeSpec = new StreamRouterSpec(); routeSpec.setPartition(partiton); routeSpec.setStreamId(partiton.getStreamId()); for (StreamWorkSlotQueue sq : ms.getQueues()) { if (sq.getTopoGroupStartIndex().containsKey(u.getTopoName())) { PolicyWorkerQueue queue = new PolicyWorkerQueue(); queue.setWorkers(sq.getWorkingSlots()); queue.setPartition(partiton); routeSpec.addQueue(queue); } } spec.addRouterSpec(routeSpec); } } } return groupSpecsMap; }
@Test public void test() { InMemMetadataServiceClient client = getSampleMetadataService(); ScheduleContextBuilder builder = new ScheduleContextBuilder(config, client); IScheduleContext context = builder.buildContext(); // assert topology usage Map<String, TopologyUsage> usages = context.getTopologyUsages(); Assert.assertEquals(1, usages.get(TOPO1).getMonitoredStream().size()); Assert.assertTrue(usages.get(TOPO1).getPolicies().contains(TEST_POLICY_1)); String alertBolt0 = TOPO1 + "-alert-" + "0"; String alertBolt1 = TOPO1 + "-alert-" + "1"; String alertBolt2 = TOPO1 + "-alert-" + "2"; for (AlertBoltUsage u : usages.get(TOPO1).getAlertUsages().values()) { if (u.getBoltId().equals(alertBolt0) || u.getBoltId().equals(alertBolt1) || u.getBoltId().equals(alertBolt2)) { Assert.assertEquals(1, u.getPolicies().size()); Assert.assertTrue(u.getPolicies().contains(TEST_POLICY_1)); Assert.assertEquals(1, u.getPartitions().size()); Assert.assertEquals(1, u.getReferQueues().size()); } } }
@Test public void test_remove_policy() { InMemMetadataServiceClient client = getSampleMetadataService(); ScheduleContextBuilder builder = new ScheduleContextBuilder(config, client); PolicyAssignment assignment1 = client.getVersionedSpec().getAssignments().get(0); IScheduleContext context = builder.buildContext(); Assert.assertTrue(context.getPolicyAssignments().containsKey(TEST_POLICY_1)); StreamWorkSlotQueue queue = SchedulerTest.getQueue(context, assignment1.getQueueId()).getRight(); client.removePolicy(0); context = builder.buildContext(); Assert.assertFalse(context.getPolicyAssignments().containsKey(TEST_POLICY_1)); WorkSlot slot = queue.getWorkingSlots().get(0); Set<String> topoPolicies = context.getTopologyUsages().get(slot.topologyName).getPolicies(); Assert.assertFalse(topoPolicies.contains(TEST_DATASOURCE_1)); Assert.assertEquals(0, topoPolicies.size()); }
TopologyUsage topologyUsage = context.getTopologyUsages().get(slot.topologyName); Set<String> topoPolicies = topologyUsage.getPolicies(); Assert.assertFalse(topoPolicies.contains(TEST_DATASOURCE_1));