workers.add(workSlot1); workers.add(workSlot2); PolicyWorkerQueue policyWorkerQueue = new PolicyWorkerQueue(workers); Assert.assertEquals(null, policyWorkerQueue.getPartition()); Assert.assertEquals(workSlot1, policyWorkerQueue.getWorkers().get(0)); Assert.assertEquals(workSlot2, policyWorkerQueue.getWorkers().get(1)); Assert.assertEquals("[(setTopologyName1:setBoltId1),(setTopologyName1:setBoltId2)]", policyWorkerQueue.toString()); PolicyWorkerQueue policyWorkerQueue1 = new PolicyWorkerQueue(); policyWorkerQueue1.setWorkers(workers); Assert.assertTrue(policyWorkerQueue.equals(policyWorkerQueue1)); Assert.assertTrue(policyWorkerQueue.hashCode() == policyWorkerQueue1.hashCode()); streamPartition.setStreamId("test"); streamPartition.setType(StreamPartition.Type.GROUPBY); policyWorkerQueue1.setPartition(streamPartition); Assert.assertFalse(policyWorkerQueue.equals(policyWorkerQueue1)); Assert.assertFalse(policyWorkerQueue.hashCode() == policyWorkerQueue1.hashCode());
private Map<String, RouterSpec> generateGroupbyMonitorMetadata() { Map<String, RouterSpec> groupSpecsMap = new HashMap<String, RouterSpec>(); for (TopologyUsage u : context.getTopologyUsages().values()) { RouterSpec spec = groupSpecsMap.get(u.getTopoName()); if (spec == null) { spec = new RouterSpec(u.getTopoName()); groupSpecsMap.put(u.getTopoName(), spec); } for (MonitoredStream ms : u.getMonitoredStream()) { // mutiple stream on the same policy group : for correlation group case: for (StreamPartition partiton : ms.getStreamGroup().getStreamPartitions()) { StreamRouterSpec routeSpec = new StreamRouterSpec(); routeSpec.setPartition(partiton); routeSpec.setStreamId(partiton.getStreamId()); for (StreamWorkSlotQueue sq : ms.getQueues()) { if (sq.getTopoGroupStartIndex().containsKey(u.getTopoName())) { PolicyWorkerQueue queue = new PolicyWorkerQueue(); queue.setWorkers(sq.getWorkingSlots()); queue.setPartition(partiton); routeSpec.addQueue(queue); } } spec.addRouterSpec(routeSpec); } } } return groupSpecsMap; }
workers.add(workSlot1); workers.add(workSlot2); PolicyWorkerQueue policyWorkerQueue = new PolicyWorkerQueue(workers); streamRouterSpec.addQueue(policyWorkerQueue); streamRouterSpec.setStreamId("streamRouterSpec"); Assert.assertEquals(2, streamRouterSpec.getTargetQueue().get(0).getWorkers().size());
/** * in correlation cases, multiple streams will go to the same queue for correlation policy. * * @param spec */ private void sanityCheck(RouterSpec spec) { Set<String> totalRequestedSlots = new HashSet<>(); for (StreamRouterSpec s : spec.getRouterSpecs()) { for (PolicyWorkerQueue q : s.getTargetQueue()) { List<String> workers = new ArrayList<>(); q.getWorkers().forEach(w -> workers.add(w.getBoltId())); totalRequestedSlots.addAll(workers); } } if (totalRequestedSlots.size() > getOutputStreamIds().size()) { String error = String.format("Requested slots are not consistent with provided slots, %s, %s", totalRequestedSlots, getOutputStreamIds()); LOG.error(error); throw new IllegalStateException(error); } }
public static StreamRouterSpec createSampleStreamRouteSpec(String streamId, String groupByField, List<String> targetEvaluatorIds) { List<WorkSlot> slots = Arrays.asList(targetEvaluatorIds.stream().map((t) -> { return new WorkSlot("sampleTopology", t); }).toArray(WorkSlot[]::new)); StreamRouterSpec streamRouteSpec = new StreamRouterSpec(); streamRouteSpec.setStreamId(streamId); streamRouteSpec.setPartition(createSampleStreamGroupbyPartition(streamId, Arrays.asList(groupByField))); streamRouteSpec.setTargetQueue(Arrays.asList(new PolicyWorkerQueue(slots))); return streamRouteSpec; }
@Test public void testRouterBoltSpec() { List<String> topics = Arrays.asList("testTopic3", "testTopic4", "testTopic5"); RouterSpec boltSpec = new RouterSpec(); for (String topic : topics) { String streamId = getStreamNameByTopic(topic); // StreamPartition, groupby col1 for stream cpuUsageStream StreamPartition sp = new StreamPartition(); sp.setStreamId(streamId); sp.setColumns(Arrays.asList("value")); sp.setType(StreamPartition.Type.GROUPBY); StreamSortSpec sortSpec = new StreamSortSpec(); sortSpec.setWindowMargin(1000); sortSpec.setWindowPeriod2(Period.seconds(10)); sp.setSortSpec(sortSpec); // set StreamRouterSpec to have 2 WorkSlot StreamRouterSpec routerSpec = new StreamRouterSpec(); routerSpec.setPartition(sp); routerSpec.setStreamId(streamId); PolicyWorkerQueue queue = new PolicyWorkerQueue(); queue.setPartition(sp); queue.setWorkers(Arrays.asList(new WorkSlot("testTopology", "alertBolt0"), new WorkSlot("testTopology", "alertBolt1"))); routerSpec.setTargetQueue(Arrays.asList(queue)); boltSpec.addRouterSpec(routerSpec); } String json = MetadataSerDeser.serialize(boltSpec); System.out.println(json); RouterSpec deserializedSpec = MetadataSerDeser.deserialize(json, RouterSpec.class); Assert.assertEquals(3, deserializedSpec.getRouterSpecs().size()); }
private List<StreamRoutePartitioner> calculatePartitioner(StreamRouterSpec streamRouterSpec, Map<String, StreamDefinition> sds, Map<StreamPartition, List<StreamRoutePartitioner>> routePartitionerMap) throws Exception { List<StreamRoutePartitioner> routePartitioners = routePartitionerMap.get(streamRouterSpec.getPartition()); if (routePartitioners == null) { routePartitioners = new ArrayList<>(); } for (PolicyWorkerQueue pwq : streamRouterSpec.getTargetQueue()) { routePartitioners.add(StreamRoutePartitionFactory.createRoutePartitioner( Lists.transform(pwq.getWorkers(), WorkSlot::getBoltId), sds.get(streamRouterSpec.getPartition().getStreamId()), streamRouterSpec.getPartition())); } return routePartitioners; }
WorkSlot worker2 = new WorkSlot("ALERT_UNIT_TOPOLOGY_APP_SANDBOX", "alertBolt2"); PolicyWorkerQueue queue1 = new PolicyWorkerQueue(); queue1.setPartition(partition); queue1.setWorkers(new ArrayList<WorkSlot>() { PolicyWorkerQueue queue2 = new PolicyWorkerQueue(); queue2.setPartition(partition); queue2.setWorkers(new ArrayList<WorkSlot>() {
@Test public void testIrregularPolicyParallelismHint() { Config config = ConfigFactory.load(); int defaultParallelism = config.getInt("coordinator.policyDefaultParallelism"); TestTopologyMgmtService mgmtService = new TestTopologyMgmtService(5, 12); InMemScheduleConext context = createScheduleContext(mgmtService); // recreate test poicy context.getPolicies().clear(); // make the hint bigger than bolt number int irregularParallelism = defaultParallelism + 2; createSamplePolicy(context, "irregularPolicy", STREAM1, irregularParallelism); GreedyPolicyScheduler ps = new GreedyPolicyScheduler(); ps.init(context, mgmtService); ScheduleState scheduled = ps.schedule(new ScheduleOption()); Assert.assertEquals(2, scheduled.getSpoutSpecs().size()); Assert.assertEquals(2, scheduled.getGroupSpecs().size()); Assert.assertEquals(2, scheduled.getAlertSpecs().size()); // assertion RouterSpec spec = scheduled.getGroupSpecs().get(TOPO1); Assert.assertTrue(spec.getRouterSpecs().size() > 0); // must be allocated for (StreamRouterSpec routerSpec : spec.getRouterSpecs()) { Assert.assertEquals(1, routerSpec.getTargetQueue().size()); // irregularParallelism is prompted to 2 * defaultParallelism = 10 Assert.assertEquals(10, routerSpec.getTargetQueue().get(0).getWorkers().size()); } }
routerSpec.setPartition(sp); routerSpec.setStreamId(streamId); PolicyWorkerQueue queue = new PolicyWorkerQueue(); queue.setPartition(sp); queue.setWorkers(Arrays.asList(new WorkSlot("testTopology", "alertBolt1"), new WorkSlot("testTopology", "alertBolt2"))); routerSpec.setTargetQueue(Collections.singletonList(queue)); boltSpec.addRouterSpec(routerSpec);
List<PolicyWorkerQueue> queues = spec.getTargetQueue(); Assert.assertEquals(1, queues.size()); Assert.assertEquals(5, queues.get(0).getWorkers().size()); for (WorkSlot slot : queues.get(0).getWorkers()) { Assert.assertEquals(topo1, slot.getTopologyName()); LOG.info(slot.getBoltId());
List<PolicyWorkerQueue> queues = spec.getTargetQueue(); Assert.assertEquals(1, queues.size()); Assert.assertEquals(5, queues.get(0).getWorkers().size()); for (WorkSlot slot : queues.get(0).getWorkers()) { Assert.assertEquals(topo1, slot.getTopologyName()); LOG.info(slot.getBoltId());