private void setup() { JetConfig cfg = new JetConfig(); cfg.setInstanceConfig(new InstanceConfig().setCooperativeThreadCount( Math.max(1, getRuntime().availableProcessors() / 2))); System.out.println("Creating Jet instance 1"); jet = Jet.newJetInstance(cfg); System.out.println("Creating Jet instance 2"); Jet.newJetInstance(cfg); System.out.println("These books will be indexed:"); buildDocumentInventory(); }
void onMemberAdded(MemberImpl addedMember) { if (addedMember.isLiteMember()) { return; } updateQuorumValues(); scheduleScaleUp(config.getInstanceConfig().getScaleUpDelayMillis()); }
private void parseInstanceConfig(Node instanceNode) { final InstanceConfig instanceConfig = jetConfig.getInstanceConfig(); for (Node node : childElements(instanceNode)) { String name = cleanNodeName(node); switch (name) { case "cooperative-thread-count": instanceConfig.setCooperativeThreadCount(intValue(node)); break; case "flow-control-period": instanceConfig.setFlowControlPeriodMs(intValue(node)); break; case "backup-count": instanceConfig.setBackupCount(intValue(node)); break; case "scale-up-delay-millis": instanceConfig.setScaleUpDelayMillis(longValue(node)); break; default: throw new AssertionError("Unrecognized XML element: " + name); } } }
private void assertJetConfig() { JetConfig jetConfig = jetInstance.getConfig(); Config hazelcastConfig = jetConfig.getHazelcastConfig(); assertHazelcastConfig(hazelcastConfig); InstanceConfig instanceConfig = jetConfig.getInstanceConfig(); assertEquals(4, instanceConfig.getBackupCount()); assertEquals(2, instanceConfig.getCooperativeThreadCount()); assertEquals(200, instanceConfig.getFlowControlPeriodMs()); assertEquals(1234, instanceConfig.getScaleUpDelayMillis()); EdgeConfig edgeConfig = jetConfig.getDefaultEdgeConfig(); assertEquals(8, edgeConfig.getQueueSize()); assertEquals(3, edgeConfig.getPacketSizeLimit()); assertEquals(5, edgeConfig.getReceiveWindowMultiplier()); assertEquals("bar", jetConfig.getProperties().getProperty("foo")); MetricsConfig metricsConfig = jetConfig.getMetricsConfig(); assertFalse(metricsConfig.isEnabled()); assertEquals(123, metricsConfig.getRetentionSeconds()); assertEquals(10, metricsConfig.getCollectionIntervalSeconds()); assertTrue(metricsConfig.isMetricsForDataStructuresEnabled()); }
public static void main(String[] args) { System.setProperty("hazelcast.logging.type", "log4j"); JetConfig config = new JetConfig(); config.getHazelcastConfig().addEventJournalConfig(new EventJournalConfig() .setMapName(TRADES_MAP_NAME) .setCapacity(TRADES_PER_SEC * 10)); config.getInstanceConfig().setCooperativeThreadCount( Math.max(1, Runtime.getRuntime().availableProcessors() / 2)); JetInstance jet = Jet.newJetInstance(config); Jet.newJetInstance(config); try { jet.newJob(buildPipeline()); TradeGenerator.generate(NUMBER_OF_TICKERS, jet.getMap(TRADES_MAP_NAME), TRADES_PER_SEC, JOB_DURATION); } finally { Jet.shutdownAll(); } }
@Override public void init(NodeEngine engine, Properties properties) { if (config == null) { throw new IllegalStateException("JetConfig is not initialized"); } jetInstance = new JetInstanceImpl((HazelcastInstanceImpl) engine.getHazelcastInstance(), config); taskletExecutionService = new TaskletExecutionService(nodeEngine, config.getInstanceConfig().getCooperativeThreadCount()); jobRepository = new JobRepository(jetInstance); jobExecutionService = new JobExecutionService(nodeEngine, taskletExecutionService, jobRepository); jobCoordinationService = new JobCoordinationService(nodeEngine, this, config, jobRepository); networking = new Networking(engine, jobExecutionService, config.getInstanceConfig().getFlowControlPeriodMs()); ClientEngineImpl clientEngine = engine.getService(ClientEngineImpl.SERVICE_NAME); ExceptionUtil.registerJetExceptions(clientEngine.getClientExceptions()); if (Boolean.parseBoolean(properties.getProperty(SHUTDOWNHOOK_ENABLED.getName()))) { logger.finest("Adding Jet shutdown hook"); Runtime.getRuntime().addShutdownHook(shutdownHookThread); } logger.info("Setting number of cooperative threads and default parallelism to " + config.getInstanceConfig().getCooperativeThreadCount()); }
) { final JetInstance instance = getJetInstance(nodeEngine); final int defaultParallelism = instance.getConfig().getInstanceConfig().getCooperativeThreadCount(); final Collection<MemberInfo> members = new HashSet<>(membersView.size()); final Address[] partitionOwners = new Address[nodeEngine.getPartitionService().getPartitionCount()];
ReceiverTasklet receiverTasklet = new ReceiverTasklet( collector, edge.getConfig().getReceiveWindowMultiplier(), getConfig().getInstanceConfig().getFlowControlPeriodMs()); addrToTasklet.put(addr, receiverTasklet); if (firstTasklet == null) {
.setBackupCount(jetConfig.getInstanceConfig().getBackupCount()) .setStatisticsEnabled(false); metadataMapConfig.getMergePolicyConfig().setPolicy(IgnoreMergingEntryMapMergePolicy.class.getName());
private String dagToJson(long jobId, JobConfig jobConfig, Data dagData) { ClassLoader classLoader = jetService.getJobExecutionService().getClassLoader(jobConfig, jobId); DAG dag = deserializeWithCustomClassLoader(nodeEngine.getSerializationService(), classLoader, dagData); int coopThreadCount = getJetInstance(nodeEngine).getConfig().getInstanceConfig().getCooperativeThreadCount(); return dag.toJson(coopThreadCount).toString(); }
private void setup() { JetConfig cfg = new JetConfig(); cfg.setInstanceConfig(new InstanceConfig().setCooperativeThreadCount( Math.max(1, getRuntime().availableProcessors() / 2))); System.out.println("Creating Jet instance 1"); jet = Jet.newJetInstance(cfg); System.out.println("Creating Jet instance 2"); Jet.newJetInstance(cfg); System.out.println("Loading The Complete Works of William Shakespeare"); try { long[] lineNum = {0}; Map<Long, String> bookLines = new HashMap<>(); Path book = Paths.get(getClass().getResource("books/shakespeare-complete-works.txt").toURI()); Files.lines(book).forEach(line -> bookLines.put(++lineNum[0], line)); jet.getMap(BOOK_LINES).putAll(bookLines); } catch (IOException | URISyntaxException e) { throw new RuntimeException(e); } }
private void setup() { JetConfig cfg = new JetConfig(); cfg.setInstanceConfig(new InstanceConfig().setCooperativeThreadCount( Math.max(1, getRuntime().availableProcessors() / 2))); System.out.println("Creating Jet instance 1"); jet = Jet.newJetInstance(cfg); System.out.println("Creating Jet instance 2"); Jet.newJetInstance(cfg); System.out.println("These books will be analyzed:"); final IMap<Long, String> docId2Name = jet.getMap(DOCID_NAME); final long[] docId = {0}; docFilenames().peek(System.out::println).forEach(line -> docId2Name.put(++docId[0], line)); }
private void run() throws Exception { JetConfig cfg = new JetConfig(); cfg.setInstanceConfig(new InstanceConfig().setCooperativeThreadCount( Math.max(1, getRuntime().availableProcessors() / 2))); try { createKafkaCluster(); fillTopics(); JetInstance instance = Jet.newJetInstance(cfg); Jet.newJetInstance(cfg); IMapJet<String, Integer> sinkMap = instance.getMap(SINK_NAME); Pipeline p = buildPipeline(); long start = System.nanoTime(); Job job = instance.newJob(p); while (true) { int mapSize = sinkMap.size(); System.out.format("Received %d entries in %d milliseconds.%n", mapSize, NANOSECONDS.toMillis(System.nanoTime() - start)); if (mapSize == MESSAGE_COUNT_PER_TOPIC * 2) { job.cancel(); break; } Thread.sleep(100); } } finally { Jet.shutdownAll(); shutdownKafkaCluster(); } }
cfg.setInstanceConfig(new InstanceConfig().setCooperativeThreadCount( Math.max(1, getRuntime().availableProcessors() / 2)));