@Override public String toString() { return String.format("StreamRouterSpec[streamId=%s,partition=%s, queue=[%s]]", this.getStreamId(), this.getPartition(), this.getTargetQueue()); }
/** * in correlation cases, multiple streams will go to the same queue for correlation policy. * * @param spec */ private void sanityCheck(RouterSpec spec) { Set<String> totalRequestedSlots = new HashSet<>(); for (StreamRouterSpec s : spec.getRouterSpecs()) { for (PolicyWorkerQueue q : s.getTargetQueue()) { List<String> workers = new ArrayList<>(); q.getWorkers().forEach(w -> workers.add(w.getBoltId())); totalRequestedSlots.addAll(workers); } } if (totalRequestedSlots.size() > getOutputStreamIds().size()) { String error = String.format("Requested slots are not consistent with provided slots, %s, %s", totalRequestedSlots, getOutputStreamIds()); LOG.error(error); throw new IllegalStateException(error); } }
private List<StreamRoutePartitioner> calculatePartitioner(StreamRouterSpec streamRouterSpec, Map<String, StreamDefinition> sds, Map<StreamPartition, List<StreamRoutePartitioner>> routePartitionerMap) throws Exception { List<StreamRoutePartitioner> routePartitioners = routePartitionerMap.get(streamRouterSpec.getPartition()); if (routePartitioners == null) { routePartitioners = new ArrayList<>(); } for (PolicyWorkerQueue pwq : streamRouterSpec.getTargetQueue()) { routePartitioners.add(StreamRoutePartitionFactory.createRoutePartitioner( Lists.transform(pwq.getWorkers(), WorkSlot::getBoltId), sds.get(streamRouterSpec.getPartition().getStreamId()), streamRouterSpec.getPartition())); } return routePartitioners; }
@Test public void testIrregularPolicyParallelismHint() { Config config = ConfigFactory.load(); int defaultParallelism = config.getInt("coordinator.policyDefaultParallelism"); TestTopologyMgmtService mgmtService = new TestTopologyMgmtService(5, 12); InMemScheduleConext context = createScheduleContext(mgmtService); // recreate test poicy context.getPolicies().clear(); // make the hint bigger than bolt number int irregularParallelism = defaultParallelism + 2; createSamplePolicy(context, "irregularPolicy", STREAM1, irregularParallelism); GreedyPolicyScheduler ps = new GreedyPolicyScheduler(); ps.init(context, mgmtService); ScheduleState scheduled = ps.schedule(new ScheduleOption()); Assert.assertEquals(2, scheduled.getSpoutSpecs().size()); Assert.assertEquals(2, scheduled.getGroupSpecs().size()); Assert.assertEquals(2, scheduled.getAlertSpecs().size()); // assertion RouterSpec spec = scheduled.getGroupSpecs().get(TOPO1); Assert.assertTrue(spec.getRouterSpecs().size() > 0); // must be allocated for (StreamRouterSpec routerSpec : spec.getRouterSpecs()) { Assert.assertEquals(1, routerSpec.getTargetQueue().size()); // irregularParallelism is prompted to 2 * defaultParallelism = 10 Assert.assertEquals(10, routerSpec.getTargetQueue().get(0).getWorkers().size()); } }
Assert.assertEquals(null, streamRouterSpec.getPartition()); Assert.assertEquals(null, streamRouterSpec.getStreamId()); Assert.assertTrue(streamRouterSpec.getTargetQueue().isEmpty()); Assert.assertEquals(1, streamRouterSpec.getTargetQueue().size()); Assert.assertEquals(2, streamRouterSpec.getTargetQueue().get(0).getWorkers().size());
Assert.assertEquals(STREAM1, spec.getStreamId()); Assert.assertEquals(1, spec.getTargetQueue().size()); List<PolicyWorkerQueue> queues = spec.getTargetQueue(); Assert.assertEquals(1, queues.size()); Assert.assertEquals(5, queues.get(0).getWorkers().size());
Assert.assertEquals(STREAM1, spec.getStreamId()); Assert.assertEquals(1, spec.getTargetQueue().size()); List<PolicyWorkerQueue> queues = spec.getTargetQueue(); Assert.assertEquals(1, queues.size()); Assert.assertEquals(5, queues.get(0).getWorkers().size());
Assert.assertTrue(spec.getTargetQueue().size() == 1);