Refine search
void maybeAddPartitions(int minPartitionNum) { KafkaZkClient zkClient = KafkaZkClient.apply(_zkConnect, JaasUtils.isZkSecurityEnabled(), ZK_SESSION_TIMEOUT_MS, ZK_CONNECTION_TIMEOUT_MS, Integer.MAX_VALUE, Time.SYSTEM, METRIC_GROUP_NAME, "SessionExpireListener"); AdminZkClient adminZkClient = new AdminZkClient(zkClient); try { scala.collection.Map<Object, scala.collection.Seq<Object>> existingAssignment = getPartitionAssignment(zkClient, _topic); int partitionNum = existingAssignment.size(); if (partitionNum < minPartitionNum) { LOG.info("MultiClusterTopicManagementService will increase partition of the topic {} " + "in cluster {} from {} to {}.", _topic, _zkConnect, partitionNum, minPartitionNum); scala.Option<scala.collection.Map<java.lang.Object, scala.collection.Seq<java.lang.Object>>> replicaAssignment = scala.Option.apply(null); scala.Option<Seq<Object>> brokerList = scala.Option.apply(null); adminZkClient.addPartitions(_topic, existingAssignment, adminZkClient.getBrokerMetadatas(RackAwareMode.Disabled$.MODULE$, brokerList), minPartitionNum, replicaAssignment, false); } } finally { zkClient.close(); } }
StringBuilder bldr = new StringBuilder(); bldr.append("{\"version\":1,\"partitions\":[\n"); for (int partition = 0; partition < partitionsToBeReassigned.size(); partition++) { bldr.append(" {\"topic\":\"").append(topic).append("\",\"partition\":").append(partition).append(",\"replicas\":["); scala.collection.Seq<Object> replicas = partitionsToBeReassigned.apply(partition); for (int replicaIndex = 0; replicaIndex < replicas.size(); replicaIndex++) { Object replica = replicas.apply(replicaIndex);
private static List<PartitionInfo> getPartitionInfo(KafkaZkClient zkClient, String topic) { scala.collection.immutable.Set<String> topicList = new scala.collection.immutable.Set.Set1<>(topic); scala.collection.Map<Object, scala.collection.Seq<Object>> partitionAssignments = zkClient.getPartitionAssignmentForTopics(topicList).apply(topic); List<PartitionInfo> partitionInfoList = new ArrayList<>(); scala.collection.Iterator<scala.Tuple2<Object, scala.collection.Seq<Object>>> it = partitionAssignments.iterator(); while (it.hasNext()) { scala.Tuple2<Object, scala.collection.Seq<Object>> scalaTuple = it.next(); Integer partition = (Integer) scalaTuple._1(); scala.Option<Object> leaderOption = zkClient.getLeaderForPartition(new TopicPartition(topic, partition)); Node leader = leaderOption.isEmpty() ? null : new Node((Integer) leaderOption.get(), "", -1); Node[] replicas = new Node[scalaTuple._2().size()]; for (int i = 0; i < replicas.length; i++) { Integer brokerId = (Integer) scalaTuple._2().apply(i); replicas[i] = new Node(brokerId, "", -1); } partitionInfoList.add(new PartitionInfo(topic, partition, leader, replicas, null)); } return partitionInfoList; }
/** * Generates a Stream that traverses the key-value pairs of a scala.collection.Map. * <p> * Only sequential operations will be efficient. * For efficient parallel operation, use the streamAccumulated method instead, but * note that this creates a new collection containing the Map's key-value pairs. * * @param coll The Map to traverse * @return A Stream view of the collection which, by default, executes sequentially. */ public static <K,V> Stream< scala.Tuple2<K, V> > stream(scala.collection.Map<K, V> coll) { return StreamSupport.stream(new StepsAnyIterator< scala.Tuple2<K, V> >(coll.iterator()), false); }
public void tryUpdateTopic(String topic) { scala.collection.mutable.Map<String, scala.collection.Map<Object, Seq<Object>>> partitionAssignmentForTopics = _zkUtils.getPartitionAssignmentForTopics(JavaConversions.asScalaBuffer(ImmutableList.of(topic))); if (partitionAssignmentForTopics.get(topic).isEmpty() || partitionAssignmentForTopics.get(topic).get().size() == 0) { LOGGER.info("try to update for topic {} but found no topic partition for it", topic); return; } synchronized (_lock) { LOGGER.info("starting to refresh for update topic {}", topic); try { _topicPartitionInfoMap.put(topic, new TopicPartition(topic, partitionAssignmentForTopics.get(topic).get().size())); } catch (Exception e) { LOGGER.warn("Failed to get topicPartition info for {} from kafka zk: {}", topic, e); } LOGGER.info("finished refreshing for updating topic {}", topic); } }
@Override public K apply(KVariable k) { if (freshVars.contains(k)) { Optional<Sort> s = k.att().getOptional(Sort.class); if (!s.isPresent()) { throw KEMException.compilerError("Fresh constant used without a declared sort.", k); } Option<KLabel> lbl = m.freshFunctionFor().get(s.get()); if (!lbl.isDefined()) { throw KEMException.compilerError("No fresh generator defined for sort " + s, k); } return KApply(lbl.get(), KApply(KLabel("_+Int_"), FRESH, KToken(offsets.get(k).toString(), Sorts.Int()))); } return super.apply(k); } }.apply(term);
/** * Get the replica assignment for a given topic partition. This information should be retrieved * from zookeeper as topic metadata that we get from kafkaConsumer.listTopic() does not specify * the preferred leader for topic partitions. * * @param tp topic partition * @return the list of brokers that host the replica */ private List<Integer> getReplicaAssignment(TopicPartition tp) { scala.collection.Map<Object, Seq<Object>> replicaAssignmentMap = getReplicaAssignmentForTopic(zkUtils, tp.topic()); scala.Option<Seq<Object>> replicasOption = replicaAssignmentMap.get(tp.partition()); Seq<Object> replicas = replicasOption.get(); List<Object> replicasList = scala.collection.JavaConverters.seqAsJavaList(replicas); return replicasList.stream().map(obj -> (Integer) obj).collect(Collectors.toList()); }
EnumSet<Fixity> innerFixity = getFixity(inner); if (inner.production().klabel().equals(outer.production().klabel()) && inner.production().klabel().isDefined() && m.attributesFor().apply(inner.production().klabel().get()).contains(Attribute.ASSOCIATIVE_KEY)) return false; if (inner instanceof Constant)
sb.append("\n"); for (Sort sort : iterable(module.definedSorts())) { Att att = module.sortAttributesFor().get(sort).getOrElse(() -> KORE.Att()); collectAttributes(attributes, att); Att att = module.sortAttributesFor().get(sort).getOrElse(() -> KORE.Att()); if (att.contains(Attribute.HOOK_KEY)) { if (collectionSorts.contains(att.get(Attribute.HOOK_KEY))) { att = att.remove(Attribute.HOOK_KEY); } else { Production concatProd = stream(module.productionsForSort().apply(sort)).filter(p -> p.att().contains("element")).findAny().get(); att = att.add("element", K.class, KApply(KLabel(concatProd.att().get("element")))); att = att.add("concat", K.class, KApply(concatProd.klabel().get())); att = att.add("unit", K.class, KApply(KLabel(concatProd.att().get("unit")))); sb.append("hooked-"); Set<KLabel> impurities = functionRules.keySet().stream().filter(lbl -> module.attributesFor().apply(lbl).contains(Attribute.IMPURE_KEY)).collect(Collectors.toSet()); impurities.addAll(ancestors(impurities, dependencies)); Set<Production> overloads = new HashSet<>(); for (Production lesser : iterable(module.overloads().elements())) { for (Production greater : iterable(module.overloads().relations().get(lesser).getOrElse(() -> Collections.<Production>Set()))) { overloads.add(greater); for (Production prod2 : iterable(module.productionsForSort().apply(prod.sort()))) { Att sortAtt = module.sortAttributesFor().get(sort).getOrElse(() -> KORE.Att()); if (!hasToken && sortAtt.contains("token")) {
private Sort getSort(ProductionReference child) { if ((child instanceof TermCons) && child.production().klabel().isDefined()) { KLabel label = child.production().klabel().get(); if (label.name().equals("#KApply")) { Term labelTerm = ((TermCons) child).get(0); Optional<KLabel> optLabel = klabelFromTerm(labelTerm); if (optLabel.isPresent() && m.productionsFor().contains(optLabel.get())) { Collection<Production> productions = mutable(m.productionsFor().get(optLabel.get()).get()); List<Term> rawArgs = lowerKList(((TermCons) child).get(1)); assert rawArgs.stream().allMatch(ProductionReference.class::isInstance);
v.setName(vs.name()); v.setIsTimeStampUnique(vs.processing().uniqueTimestamp()); if (vs.poolSize().isDefined()) v.setLifeCyclePoolSize(((Integer)vs.poolSize().get())); if (vs.processing().rate().isDefined()) v.setOutputStreamRate(((Integer)vs.processing().rate().get())); v.setPriority(vs.priority()); Map<String,String> init=vs.processing().initParams(); ArrayList<KeyValue> ini=new ArrayList<KeyValue>(); Iterable<String> initkeys=JavaConversions.asJavaIterable(init.keys()); for (String ik:initkeys){ logger.trace("keys:"+ik); ini.add(new KeyValueImp(ik.toLowerCase(),init.apply(ik)));
public static void main(String[] args) throws Exception { CommandLine commandLine = parseCommandLine(args); String zookeeper = commandLine.getOptionValue(ZOOKEEPER); ZkUtils zkUtils = KafkaUtils.getZkUtils(zookeeper); Seq<String> topicsSeq = zkUtils.getAllTopics(); List<String> topics = scala.collection.JavaConverters.seqAsJavaList(topicsSeq); scala.collection.mutable.Map<String, scala.collection.Map<Object, Seq<Object>>> partitionAssignments = zkUtils.getPartitionAssignmentForTopics(topicsSeq); Map<String, Integer> replicationFactors = new HashMap<>(); Map<String, Integer> partitionCounts = new HashMap<>(); topics.stream().forEach(topic -> { int partitionCount = partitionAssignments.get(topic).get().size(); int factor = partitionAssignments.get(topic).get().head()._2().size(); partitionCounts.put(topic, partitionCount); replicationFactors.put(topic, factor); }); List<PartitionInfo> urps = KafkaClusterManager.getUnderReplicatedPartitions( zookeeper, SecurityProtocol.PLAINTEXT, null, topics, partitionAssignments, replicationFactors, partitionCounts); for (PartitionInfo partitionInfo : urps) { LOG.info("under-replicated : {}", partitionInfo); } } }
private void forEachKLabel(Consumer<Tuple2<KLabel, Long>> action) { for (KLabel label : iterable(mainModule.definedKLabels())) { if (ConvertDataStructureToLookup.isLookupKLabel(label) || label.name().equals("#KToken")) continue; stream(mainModule.productionsFor().apply(label)).map(p -> Tuple2.apply(p.klabel().get(), stream(p.items()).filter(pi -> pi instanceof NonTerminal).count())).distinct().forEach(action); } }
if (kseq.items().size() == 1 && kseq.items().get(0) instanceof KApply) { KApply kapp = (KApply) kseq.items().get(0); if (mainModule.attributesFor().apply(kapp.klabel()).contains(Attribute.FUNCTION_KEY)) { functionRules.put(kapp.klabel(), r); klabelsForEachPredicate = new HashMap<>(); for (KLabel functionLabel : rules.keySet()) { if (mainModule.attributesFor().get(functionLabel).getOrElse(() -> Att()).contains("klabelPredicate")) { klabelsForEachPredicate.put(functionLabel.name(), computeKLabelsForPredicate(functionRules.get(functionLabel))); Set<KLabel> impurities = functions.stream().filter(lbl -> mainModule.attributesFor().apply(lbl).contains(Attribute.IMPURE_KEY)).collect(Collectors.toSet()); impurities.addAll(ancestors(impurities, dependencies)); constants = functions.stream().filter(lbl -> !impurities.contains(lbl) && stream(mainModule.productionsFor().apply(lbl)).filter(p -> p.arity() == 0).findAny().isPresent()).collect(Collectors.toSet()); String hook = mainModule.attributesFor().get(functionLabel).getOrElse(() -> Att()).<String>getOptional(Attribute.HOOK_KEY).orElse("."); if (hook.equals("KREFLECTION.fresh")) { sb.append(conn).append("freshFunction (sort: string) (config: k) (counter: Z.t) : k = match sort with \n"); for (Sort sort : iterable(mainModule.freshFunctionFor().keys())) { sb.append("| \"").append(sort.toString()).append("\" -> ("); KLabel freshFunction = mainModule.freshFunctionFor().apply(sort); encodeStringToFunction(sb, freshFunction); sb.append(" ([Int counter]) config (-1))\n"); sb.append(conn); String functionName; if (mainModule.attributesFor().apply(functionLabel).contains("memo")) { functionName = encodeStringToMemoFunction(sb, functionLabel); } else {
private void verifySchemaTopic() { Set<String> topics = new HashSet<String>(); topics.add(topic); // check # partition and the replication factor scala.collection.Map partitionAssignment = zkUtils.getPartitionAssignmentForTopics( JavaConversions.asScalaSet(topics).toSeq()) .get(topic).get(); if (partitionAssignment.size() != 1) { log.warn("The schema topic " + topic + " should have only 1 partition."); } if (((Seq) partitionAssignment.get(0).get()).size() < desiredReplicationFactor) { log.warn("The replication factor of the schema topic " + topic + " is less than the " + "desired one of " + desiredReplicationFactor + ". If this is a production " + "environment, it's crucial to add more brokers and increase the replication " + "factor of the topic."); } // check the retention policy Properties prop = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), topic); String retentionPolicy = prop.getProperty(LogConfig.CleanupPolicyProp()); if (retentionPolicy == null || "compact".compareTo(retentionPolicy) != 0) { log.warn("The retention policy of the schema topic " + topic + " may be incorrect. " + "Please configure it with compact."); } }
List<K> elementsLeft = new ArrayList<K>(); List<K> elementsRight = new ArrayList<K>(); KLabel elementLabel = KLabel(m.attributesFor().apply(collectionLabel).<String>get("element")); boolean isRight = false; // true for components later than the frame variable. K tail; if (frame == null) { tail = KORE.KApply(KLabel(m.attributesFor().apply(collectionLabel).<String>get(Attribute.UNIT_KEY))); } else { tail = frame; list = newDotVariable(m.productionsFor().get(collectionLabel).get().head().sort()); KToken(Integer.toString(elementsRight.size()), Sorts.Int())))); } else { KLabel unit = KLabel(m.attributesFor().apply(collectionLabel).<String>get("unit")); KLabel elementWrapper = KLabel(m.attributesFor().apply(collectionLabel).<String>get("element")); for (int i = 0; i < elementsLeft.size(); i++) { state.add(KORE.KApply(
Sort.of(module.productionsFor().get(klabel).get().head().sort()), (KLabelConstant) convertedKLabel, KLabelConstant.of(org.kframework.backend.java.kil.KLabel.parse(module.attributesFor().get(klabel).get().<String>get(Att.unit())), global.getDefinition()), global); BuiltinList.Builder builder = BuiltinList.builder( Sort.of(stream(module.productionsFor().toStream()).filter(t -> t._1.equals(assocKLabelForUnit.get())).findAny().get()._2.head().sort()), KLabelConstant.of(assocKLabelForUnit.get(), global.getDefinition()), (KLabelConstant) convertedKLabel,
@Override public void apply(KToken k) { sb.append("\\dv{"); convert(k.sort(), false); sb.append("}("); if (module.sortAttributesFor().get(k.sort()).getOrElse(() -> Att.empty()).getOptional("hook").orElse("").equals("STRING.String")) { sb.append(k.s()); } else { sb.append(StringUtil.enquoteKString(k.s())); } sb.append(")"); }