public Map<StreamPartition, List<StreamRouterSpec>> makeSRS() { Map<StreamPartition, List<StreamRouterSpec>> newSRS = new HashMap<>(); this.getRouterSpecs().forEach(t -> { if (!newSRS.containsKey(t.getPartition())) { newSRS.put(t.getPartition(), new ArrayList<>()); } newSRS.get(t.getPartition()).add(t); }); return newSRS; }
/** * in correlation cases, multiple streams will go to the same queue for correlation policy. * * @param spec */ private void sanityCheck(RouterSpec spec) { Set<String> totalRequestedSlots = new HashSet<>(); for (StreamRouterSpec s : spec.getRouterSpecs()) { for (PolicyWorkerQueue q : s.getTargetQueue()) { List<String> workers = new ArrayList<>(); q.getWorkers().forEach(w -> workers.add(w.getBoltId())); totalRequestedSlots.addAll(workers); } } if (totalRequestedSlots.size() > getOutputStreamIds().size()) { String error = String.format("Requested slots are not consistent with provided slots, %s, %s", totalRequestedSlots, getOutputStreamIds()); LOG.error(error); throw new IllegalStateException(error); } }
public Map<StreamPartition, StreamSortSpec> makeSSS() { Map<StreamPartition, StreamSortSpec> newSSS = new HashMap<>(); this.getRouterSpecs().forEach(t -> { if (t.getPartition().getSortSpec() != null) { newSSS.put(t.getPartition(), t.getPartition().getSortSpec()); } }); return newSSS; }
@Test public void testIrregularPolicyParallelismHint() { Config config = ConfigFactory.load(); int defaultParallelism = config.getInt("coordinator.policyDefaultParallelism"); TestTopologyMgmtService mgmtService = new TestTopologyMgmtService(5, 12); InMemScheduleConext context = createScheduleContext(mgmtService); // recreate test poicy context.getPolicies().clear(); // make the hint bigger than bolt number int irregularParallelism = defaultParallelism + 2; createSamplePolicy(context, "irregularPolicy", STREAM1, irregularParallelism); GreedyPolicyScheduler ps = new GreedyPolicyScheduler(); ps.init(context, mgmtService); ScheduleState scheduled = ps.schedule(new ScheduleOption()); Assert.assertEquals(2, scheduled.getSpoutSpecs().size()); Assert.assertEquals(2, scheduled.getGroupSpecs().size()); Assert.assertEquals(2, scheduled.getAlertSpecs().size()); // assertion RouterSpec spec = scheduled.getGroupSpecs().get(TOPO1); Assert.assertTrue(spec.getRouterSpecs().size() > 0); // must be allocated for (StreamRouterSpec routerSpec : spec.getRouterSpecs()) { Assert.assertEquals(1, routerSpec.getTargetQueue().size()); // irregularParallelism is prompted to 2 * defaultParallelism = 10 Assert.assertEquals(10, routerSpec.getTargetQueue().get(0).getWorkers().size()); } }
Assert.assertEquals(2, joinSpout.getStreamRepartitionMetadataMap().get(TEST_TOPIC).size()); Assert.assertEquals(2, groupSpec.getRouterSpecs().size());
Assert.assertEquals(1, spec.getRouterSpecs().size()); StreamRouterSpec routeSpec = spec.getRouterSpecs().get(0); Assert.assertEquals(STREAM2, routeSpec.getStreamId()); Assert.assertEquals(Arrays.asList("col1"), routeSpec.getPartition().getColumns());
String topo1 = gspec.getTopologyName(); LOG.info("group spec topology name:", topo1); List<StreamRouterSpec> routeSpecs = gspec.getRouterSpecs(); Assert.assertEquals(1, routeSpecs.size()); for (StreamRouterSpec spec : routeSpecs) { List<StreamRouterSpec> routeSpecs = gs2.getRouterSpecs(); Assert.assertEquals(0, routeSpecs.size());
String topo1 = gspec.getTopologyName(); LOG.info("group spec topology name:", topo1); List<StreamRouterSpec> routeSpecs = gspec.getRouterSpecs(); Assert.assertEquals(1, routeSpecs.size()); for (StreamRouterSpec spec : routeSpecs) { List<StreamRouterSpec> routeSpecs = gs2.getRouterSpecs(); Assert.assertEquals(0, routeSpecs.size());
ScheduleState state = ps.getState(); Assert.assertTrue(state.getGroupSpecs().get("topo2").getRouterSpecs().size() == 0); state = ps.getState(); for(StreamRouterSpec spec : state.getGroupSpecs().get("topo2").getRouterSpecs()) { if (spec.getStreamId().equals(STREAM1)) { Assert.assertTrue(spec.getTargetQueue().size() == 1);
@Test public void testRouterBoltSpec() { List<String> topics = Arrays.asList("testTopic3", "testTopic4", "testTopic5"); RouterSpec boltSpec = new RouterSpec(); for (String topic : topics) { String streamId = getStreamNameByTopic(topic); // StreamPartition, groupby col1 for stream cpuUsageStream StreamPartition sp = new StreamPartition(); sp.setStreamId(streamId); sp.setColumns(Arrays.asList("value")); sp.setType(StreamPartition.Type.GROUPBY); StreamSortSpec sortSpec = new StreamSortSpec(); sortSpec.setWindowMargin(1000); sortSpec.setWindowPeriod2(Period.seconds(10)); sp.setSortSpec(sortSpec); // set StreamRouterSpec to have 2 WorkSlot StreamRouterSpec routerSpec = new StreamRouterSpec(); routerSpec.setPartition(sp); routerSpec.setStreamId(streamId); PolicyWorkerQueue queue = new PolicyWorkerQueue(); queue.setPartition(sp); queue.setWorkers(Arrays.asList(new WorkSlot("testTopology", "alertBolt0"), new WorkSlot("testTopology", "alertBolt1"))); routerSpec.setTargetQueue(Arrays.asList(queue)); boltSpec.addRouterSpec(routerSpec); } String json = MetadataSerDeser.serialize(boltSpec); System.out.println(json); RouterSpec deserializedSpec = MetadataSerDeser.deserialize(json, RouterSpec.class); Assert.assertEquals(3, deserializedSpec.getRouterSpecs().size()); }