OpResult result = new OpResult(); try { for (String key : state.getSpoutSpecs().keySet()) { SpoutSpec spoutSpec = state.getSpoutSpecs().get(key); addOneSpoutSpec(spoutSpec); for (String key : state.getAlertSpecs().keySet()) { AlertBoltSpec alertBoltSpec = state.getAlertSpecs().get(key); addOne(alertSpecs, alertBoltSpec); for (String key : state.getGroupSpecs().keySet()) { RouterSpec groupSpec = state.getGroupSpecs().get(key); addOne(groupSpecs, groupSpec); for (String key : state.getPublishSpecs().keySet()) { PublishSpec publishSpec = state.getPublishSpecs().get(key); addOne(publishSpecs, publishSpec); for (VersionedPolicyDefinition policySnapshot : state.getPolicySnapshots()) { addOne(policySnapshots, policySnapshot); for (VersionedStreamDefinition streamSnapshot : state.getStreamSnapshots()) { addOne(streamSnapshots, streamSnapshot); for (MonitoredStream monitoredStream : state.getMonitoredStreams()) { addOne(monitoredStreams, monitoredStream); for (PolicyAssignment assignment : state.getAssignments()) { addOne(assignments, assignment);
private static ScheduleState createScheduleState() { ScheduleState ss = new ScheduleState(); ss.setVersion(V1); ss.getMonitoredStreams().add(createMonitoredStream()); ss.getAssignments().add(createAssignment()); return ss; }
public static void postSchedule(IMetadataServiceClient client, ScheduleState state, ConfigBusProducer producer) { // persist state client.addScheduleState(state); // notify ConfigValue value = new ConfigValue(); value.setValue(state.getVersion()); value.setValueVersionId(true); for (String topo : state.getSpoutSpecs().keySet()) { producer.send(MessageFormat.format(ZK_ALERT_CONFIG_SPOUT, topo), value); } for (String topo : state.getGroupSpecs().keySet()) { producer.send(MessageFormat.format(ZK_ALERT_CONFIG_ROUTER, topo), value); } for (String topo : state.getAlertSpecs().keySet()) { producer.send(MessageFormat.format(ZK_ALERT_CONFIG_ALERT, topo), value); } for (String topo : state.getPublishSpecs().keySet()) { producer.send(MessageFormat.format(ZK_ALERT_CONFIG_PUBLISHER, topo), value); } }
return; Map<String, StreamDefinition> sds = getStreams(state.getStreamSnapshots()); switch (type) { case ALERT_BOLT: AlertBoltSpec alertSpec = state.getAlertSpecs().get(topologyId); if (alertSpec == null) { LOG.error(" alert spec for version {} not found for topology {} !", version, topologyId); } else { prePopulate(alertSpec, state.getPolicySnapshots()); notifyAlertBolt(alertSpec, sds); PublishSpec pubSpec = state.getPublishSpecs().get(topologyId); if (pubSpec == null) { LOG.error(" alert spec for version {} not found for topology {} !", version, topologyId); } else { notifyAlertPublishBolt(pubSpec, sds); if (state.getAlertSpecs().get(topologyId) != null) { notifyAlertPublishBolt(listToMap(state.getPolicySnapshots()), sds); SpoutSpec spoutSpec = state.getSpoutSpecs().get(topologyId); if (spoutSpec == null) { LOG.error(" alert spec for version {} not found for topology {} !", version, topologyId); RouterSpec gSpec = state.getGroupSpecs().get(topologyId); if (gSpec == null) { LOG.error(" alert spec for version {} not found for topology {} !", version, topologyId);
@Test public void testIrregularPolicyParallelismHint() { Config config = ConfigFactory.load(); int defaultParallelism = config.getInt("coordinator.policyDefaultParallelism"); TestTopologyMgmtService mgmtService = new TestTopologyMgmtService(5, 12); InMemScheduleConext context = createScheduleContext(mgmtService); // recreate test poicy context.getPolicies().clear(); // make the hint bigger than bolt number int irregularParallelism = defaultParallelism + 2; createSamplePolicy(context, "irregularPolicy", STREAM1, irregularParallelism); GreedyPolicyScheduler ps = new GreedyPolicyScheduler(); ps.init(context, mgmtService); ScheduleState scheduled = ps.schedule(new ScheduleOption()); Assert.assertEquals(2, scheduled.getSpoutSpecs().size()); Assert.assertEquals(2, scheduled.getGroupSpecs().size()); Assert.assertEquals(2, scheduled.getAlertSpecs().size()); // assertion RouterSpec spec = scheduled.getGroupSpecs().get(TOPO1); Assert.assertTrue(spec.getRouterSpecs().size() > 0); // must be allocated for (StreamRouterSpec routerSpec : spec.getRouterSpecs()) { Assert.assertEquals(1, routerSpec.getTargetQueue().size()); // irregularParallelism is prompted to 2 * defaultParallelism = 10 Assert.assertEquals(10, routerSpec.getTargetQueue().get(0).getWorkers().size()); } }
@Test public void testDataSources() throws Exception { InMemScheduleConext context = loadContext("/multi/"); TestTopologyMgmtService mgmtService = new TestTopologyMgmtService(4, 10); GreedyPolicyScheduler ps = new GreedyPolicyScheduler(); ps.init(context, mgmtService); ScheduleState state = ps.schedule(new ScheduleOption()); Assert.assertNotNull(state); Assert.assertEquals(2, state.getAssignments().size()); Assert.assertEquals(1, state.getAlertSpecs().size()); Assert.assertEquals(10, state.getAlertSpecs().get("alertUnitTopology_1").getBoltPolicyIdsMap().size()); }
@Override public synchronized OpResult addScheduleState(ScheduleState state) { // FIXME : might concurrent issue String toRemove = null; if (scheduleStates.size() > maxScheduleState) { toRemove = scheduleStates.firstKey(); } scheduleStates.put(state.getVersion(), state); if (toRemove != null) { scheduleStates.remove(toRemove); } OpResult result = new OpResult(); result.code = 200; result.message = "OK"; return result; }
@Override public List<SpoutSpec> listSpoutMetadata() { ScheduleState state = getVersionedSpec(); return new ArrayList<>(state.getSpoutSpecs().values()); }
assignments = listToMap(state == null ? new ArrayList<PolicyAssignment>() : detectAssignmentsChange(state.getAssignments(), state)); monitoredStreamMap = listToMap(state == null ? new ArrayList<MonitoredStream>() : detectMonitoredStreams(state.getMonitoredStreams()));
@Test public void test_remove_policy() { InMemMetadataServiceClient client = getSampleMetadataService(); ScheduleContextBuilder builder = new ScheduleContextBuilder(config, client); PolicyAssignment assignment1 = client.getVersionedSpec().getAssignments().get(0); IScheduleContext context = builder.buildContext(); Assert.assertTrue(context.getPolicyAssignments().containsKey(TEST_POLICY_1)); StreamWorkSlotQueue queue = SchedulerTest.getQueue(context, assignment1.getQueueId()).getRight(); client.removePolicy(0); context = builder.buildContext(); Assert.assertFalse(context.getPolicyAssignments().containsKey(TEST_POLICY_1)); WorkSlot slot = queue.getWorkingSlots().get(0); Set<String> topoPolicies = context.getTopologyUsages().get(slot.topologyName).getPolicies(); Assert.assertFalse(topoPolicies.contains(TEST_DATASOURCE_1)); Assert.assertEquals(0, topoPolicies.size()); }
IMetadataServiceClient client = new MetadataServiceClientImpl(this.getConfig()); String topologyId = spec.getTopologyName(); AlertBoltSpec latestSpec = client.getVersionedSpec().getAlertSpecs().get(topologyId); if (latestSpec != null) { spec = latestSpec;
ScheduleState state = ps.getState(); Assert.assertTrue(state.getGroupSpecs().get("topo2").getRouterSpecs().size() == 0); state = ps.getState(); for(StreamRouterSpec spec : state.getGroupSpecs().get("topo2").getRouterSpecs()) { if (spec.getStreamId().equals(STREAM1)) { Assert.assertTrue(spec.getTargetQueue().size() == 1);
private List<PolicyAssignment> detectAssignmentsChange(List<PolicyAssignment> list, ScheduleState state) { // FIXME: duplicated build map ? Map<String, StreamWorkSlotQueue> queueMap = new HashMap<String, StreamWorkSlotQueue>(); for (MonitoredStream ms : state.getMonitoredStreams()) { for (StreamWorkSlotQueue q : ms.getQueues()) { queueMap.put(q.getQueueId(), q); } } List<PolicyAssignment> result = new ArrayList<PolicyAssignment>(list); Iterator<PolicyAssignment> paIt = result.iterator(); while (paIt.hasNext()) { PolicyAssignment assignment = paIt.next(); if (!policies.containsKey(assignment.getPolicyName())) { LOG.info("Policy assignment {} 's policy not found, this assignment will be removed!", assignment); paIt.remove(); } else { StreamWorkSlotQueue queue = queueMap.get(assignment.getQueueId()); if (queue == null || policies.get(assignment.getPolicyName()).getParallelismHint() != queue.getQueueSize()) { // queue not found or policy has hint not equal to queue (possible a poilcy update) LOG.info("Policy assignment {} 's policy doesnt match queue: {}!", assignment, queue); paIt.remove(); } } } return result; }
@Ignore @Test public void addScheduleState() throws Exception { ConfigFactory.invalidateCaches(); System.setProperty("config.resource", "/test-application.conf"); Config config = ConfigFactory.load("test-application.conf").getConfig("coordinator"); MetadataServiceClientImpl client = new MetadataServiceClientImpl(config); ScheduleState ss = new ScheduleState(); ss.setVersion("spec_version_1463764252582"); client.addScheduleState(ss); client.close(); ss.setVersion("spec_version_1464764252582"); ZKConfig zkConfig = ZKConfigBuilder.getZKConfig(config); ConfigBusProducer producer = new ConfigBusProducer(zkConfig); Coordinator.postSchedule(client, ss, producer); } }
public ScheduleState generate(List<WorkItem> expandworkSets) { // topologyId -> SpoutSpec Map<String, SpoutSpec> topoSpoutSpecsMap = generateSpoutMonitorMetadata(); // grp-by meta spec(sort & grp) Map<String, RouterSpec> groupSpecsMap = generateGroupbyMonitorMetadata(); // alert bolt spec Map<String, AlertBoltSpec> alertSpecsMap = generateAlertMonitorMetadata(); Map<String, PublishSpec> publishSpecsMap = generatePublishMetadata(); String uniqueVersion = generateVersion(); ScheduleState status = new ScheduleState(uniqueVersion, topoSpoutSpecsMap, groupSpecsMap, alertSpecsMap, publishSpecsMap, context.getPolicyAssignments().values(), context.getMonitoredStreams().values(), context.getPolicies().values(), context.getStreamSchemas().values()); return status; }
private void assertFirstPolicyScheduled(IScheduleContext context, ScheduleState status) { String version = status.getVersion(); Iterator<SpoutSpec> it = status.getSpoutSpecs().values().iterator(); Iterator<RouterSpec> gsit = status.getGroupSpecs().values().iterator(); Assert.assertEquals(2, status.getGroupSpecs().values().size()); Assert.assertTrue(gsit.hasNext()); RouterSpec gspec = gsit.next(); Assert.assertEquals(2, status.getAlertSpecs().values().size()); Iterator<AlertBoltSpec> asit = status.getAlertSpecs().values().iterator();
SpoutSpec joinSpout = state.getSpoutSpecs().get(joinTopo); RouterSpec groupSpec = state.getGroupSpecs().get(joinTopo); AlertBoltSpec alertSpec = state.getAlertSpecs().get(joinTopo);
@Override public void addScheduleState(ScheduleState state) { scheduleStates.put(state.getVersion(), state); }
@Test public void test01_simple() throws Exception { GreedyPolicyScheduler ps = new GreedyPolicyScheduler(); TestTopologyMgmtService mgmtService = createMgmtService(); IScheduleContext context = createScheduleContext(mgmtService); ps.init(context, mgmtService); ps.schedule(new ScheduleOption()); ScheduleState status = ps.getState(); context = ps.getContext(); // context updated! Map<String, SpoutSpec> spec = status.getSpoutSpecs(); LOG.info(mapper.writeValueAsString(spec)); Assert.assertEquals(2, spec.size()); Assert.assertTrue(spec.containsKey(TOPO1)); assertFirstPolicyScheduled(context, status); }
@Test public void test_changed_policy_parallelism() { InMemMetadataServiceClient client = getSampleMetadataService(); ScheduleContextBuilder builder = new ScheduleContextBuilder(config, client); PolicyAssignment assignment1 = client.getVersionedSpec().getAssignments().get(0); IScheduleContext context = builder.buildContext(); Assert.assertTrue(context.getPolicyAssignments().containsKey(TEST_POLICY_1)); StreamWorkSlotQueue queue = SchedulerTest.getQueue(context, assignment1.getQueueId()).getRight(); PolicyDefinition pd1 = client.listPolicies().get(0); pd1.setParallelismHint(4); // default queue is 5 , change to smaller, same like change bigger context = builder.buildContext(); Assert.assertFalse(context.getPolicyAssignments().values().iterator().hasNext()); //PolicyAssignment assignmentNew = context.getPolicyAssignments().values().iterator().next(); //StreamWorkSlotQueue queueNew = SchedulerTest.getQueue(context, assignmentNew.getQueueId()).getRight(); //Assert.assertNotNull(queueNew); // just to make sure queueNew is present //Assert.assertEquals(queue.getQueueId(), queueNew.getQueueId()); // default queue is 5 , change to bigger 6, policy assignment removed pd1.setParallelismHint(queue.getQueueSize() + 1); context = builder.buildContext(); Assert.assertFalse(context.getPolicyAssignments().values().iterator().hasNext()); }