int instanceCount = entry.getValue().getInstances(); for (int instanceId = 0; instanceId < instanceCount; instanceId++) { flowlets.put(entry.getKey(), instanceId,
String flowletId = entry.getKey(); long groupId = FlowUtils.generateConsumerGroupId(program, flowletId); int instances = entry.getValue().getInstances();
@Override public ProgramController run(Program program, ProgramOptions options) { // Extract and verify parameters FlowSpecification flowSpec = program.getSpecification(); ProgramType processorType = program.getType(); Preconditions.checkNotNull(processorType, "Missing processor type."); Preconditions.checkArgument(processorType == ProgramType.FLOW, "Only FLOW process type is supported."); Preconditions.checkNotNull(flowSpec, "Missing FlowSpecification for %s", program.getName()); for (FlowletDefinition flowletDefinition : flowSpec.getFlowlets().values()) { int maxInstances = flowletDefinition.getFlowletSpec().getMaxInstances(); Preconditions.checkArgument(flowletDefinition.getInstances() <= maxInstances, "Flowlet %s can have a maximum of %s instances", flowletDefinition.getFlowletSpec().getName(), maxInstances); } try { // Launch flowlet program runners RunId runId = RunIds.generate(); programOptions.put(runId, options); Multimap<String, QueueName> consumerQueues = FlowUtils.configureQueue(program, flowSpec, queueAdmin); final Table<String, Integer, ProgramController> flowlets = createFlowlets(program, runId, flowSpec); return new FlowProgramController(flowlets, runId, program, flowSpec, consumerQueues, discoveryServiceClient); } catch (Exception e) { throw Throwables.propagate(e); } }
@Override protected ProgramController launch(Program program, ProgramOptions options, File hConfFile, File cConfFile, ApplicationLauncher launcher) { // Extract and verify parameters FlowSpecification flowSpec = program.getSpecification(); ProgramType processorType = program.getType(); Preconditions.checkNotNull(processorType, "Missing processor type."); Preconditions.checkArgument(processorType == ProgramType.FLOW, "Only FLOW process type is supported."); try { Preconditions.checkNotNull(flowSpec, "Missing FlowSpecification for %s", program.getName()); for (FlowletDefinition flowletDefinition : flowSpec.getFlowlets().values()) { int maxInstances = flowletDefinition.getFlowletSpec().getMaxInstances(); Preconditions.checkArgument(flowletDefinition.getInstances() <= maxInstances, "Flowlet %s can have a maximum of %s instances", flowletDefinition.getFlowletSpec().getName(), maxInstances); } LOG.info("Configuring flowlets queues"); Multimap<String, QueueName> flowletQueues = FlowUtils.configureQueue(program, flowSpec, queueAdmin); // Launch flowlet program runners LOG.info("Launching distributed flow: " + program.getName() + ":" + flowSpec.getName()); TwillController controller = launcher.launch(new FlowTwillApplication(program, flowSpec, hConfFile, cConfFile, eventHandler)); DistributedFlowletInstanceUpdater instanceUpdater = new DistributedFlowletInstanceUpdater(program, controller, queueAdmin, flowletQueues); return new FlowTwillProgramController(program.getName(), controller, instanceUpdater).startListen(); } catch (Exception e) { throw Throwables.propagate(e); } }
.setVirtualCores(flowletSpec.getResources().getVirtualCores()) .setMemory(flowletSpec.getResources().getMemoryMB(), ResourceSpecification.SizeUnit.MEGA) .setInstances(flowletDefinition.getInstances()) .build();