@Override public synchronized Option<URL> resolve(Path remoteFile) { Option<URL> resolved = Option.apply(paths.get(remoteFile)); return resolved; }
Map<Integer, String> rackByBroker = new HashMap<>(); for (BrokerMetadata bm : JavaConversions.seqAsJavaList(AdminUtils.getBrokerMetadatas(zkUtils, RackAwareMode.Enforced$.MODULE$, Option.empty()))) { String rack = bm.rack().isEmpty() ? String.valueOf(bm.id()) : bm.rack().get(); brokersByRack.putIfAbsent(rack, new ArrayList<>()); brokersByRack.get(rack).add(bm.id());
/** * Build a list of URIs for providing custom artifacts to Mesos tasks. * @param uris a comma delimited optional string listing artifact URIs */ public static List<String> buildUris(Option<String> uris) { if (uris.isEmpty()) { return Collections.emptyList(); } else { List<String> urisList = new ArrayList<>(); for (String uri : uris.get().split(",")) { urisList.add(uri.trim()); } return urisList; } }
/** * Get the persisted framework ID. * @return the current ID or empty if none is yet persisted. * @throws Exception on ZK failures, interruptions. */ @Override public Option<Protos.FrameworkID> getFrameworkID() throws Exception { synchronized (startStopLock) { verifyIsRunning(); Option<Protos.FrameworkID> frameworkID; byte[] value = frameworkIdInZooKeeper.getValue(); if (value.length == 0) { frameworkID = Option.empty(); } else { frameworkID = Option.apply(Protos.FrameworkID.newBuilder().setValue(new String(value, ConfigConstants.DEFAULT_CHARSET)).build()); } return frameworkID; } }
final String akkaHostname = AkkaUtils.getAddress(actorSystem).host().get(); final int akkaPort = (Integer) AkkaUtils.getAddress(actorSystem).port().get(); highAvailabilityServices, metricRegistry, webMonitor == null ? Option.empty() : Option.apply(webMonitor.getRestAddress()), new Some<>(JobMaster.JOB_MANAGER_NAME), Option.<String>empty(), getJobManagerClass(), getArchivistClass())._1();
@Override public Node nextNode(Long capability, Long persistentCapability){ Option<com.linkedin.norbert.cluster.Node> node = loadBalancer.nextNode(new Some<Long>(capability.longValue()), new Some<Long>(persistentCapability.longValue())); if(node.isDefined()) return JavaNode.apply(node.get()); else return null; }
public Term apply(TermCons tc) { for (int i = 0, j = 0; i < tc.production().items().size(); i++) { if (tc.production().items().apply(i) instanceof NonTerminal) { if (tc.production().klabel().isDefined() && (tc.production().klabel().get().name().equals("#SyntacticCast") || tc.production().klabel().get().name().startsWith("#SemanticCastTo") || tc.production().klabel().get().name().equals("#InnerCast")) || (isFunctionRule(tc) && j == 0)) { Term t = tc.get(0); new CollectUndeclaredVariables2(getSortOfCast(tc)).apply(t); j++; } else { Term t = tc.get(j); new CollectUndeclaredVariables2(((NonTerminal) tc.production().items().apply(i)).sort()).apply(t); j++; } } } return super.apply(tc); }
/** * If option is defined, applies doWithOption function, else returns Option.none() */ final public static <U, T> Option<U> map(Function<T, U> doWithOption, Option<T> option) { if (option.isDefined()) { T t = option.get(); return Option.apply(doWithOption.apply(t)); } else { return Option.empty(); } } }
private static List<PartitionInfo> getPartitionInfo(KafkaZkClient zkClient, String topic) { scala.collection.immutable.Set<String> topicList = new scala.collection.immutable.Set.Set1<>(topic); scala.collection.Map<Object, scala.collection.Seq<Object>> partitionAssignments = zkClient.getPartitionAssignmentForTopics(topicList).apply(topic); List<PartitionInfo> partitionInfoList = new ArrayList<>(); scala.collection.Iterator<scala.Tuple2<Object, scala.collection.Seq<Object>>> it = partitionAssignments.iterator(); while (it.hasNext()) { scala.Tuple2<Object, scala.collection.Seq<Object>> scalaTuple = it.next(); Integer partition = (Integer) scalaTuple._1(); scala.Option<Object> leaderOption = zkClient.getLeaderForPartition(new TopicPartition(topic, partition)); Node leader = leaderOption.isEmpty() ? null : new Node((Integer) leaderOption.get(), "", -1); Node[] replicas = new Node[scalaTuple._2().size()]; for (int i = 0; i < replicas.length; i++) { Integer brokerId = (Integer) scalaTuple._2().apply(i); replicas[i] = new Node(brokerId, "", -1); } partitionInfoList.add(new PartitionInfo(topic, partition, leader, replicas, null)); } return partitionInfoList; }
@Override public String getSparkUiWebUrl() { return getOrCreate().sparkContext().uiWebUrl().get(); }
@Override public int getLeaderToShutDown(String topic) throws Exception { ZkUtils zkUtils = getZkUtils(); try { PartitionMetadata firstPart = null; do { if (firstPart != null) { LOG.info("Unable to find leader. error code {}", firstPart.errorCode()); // not the first try. Sleep a bit Thread.sleep(150); } Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionsMetadata(); firstPart = partitionMetadata.head(); } while (firstPart.errorCode() != 0); return firstPart.leader().get().id(); } finally { zkUtils.close(); } }
public static StreamSource source(SourceConf sc){ StreamSource s = new StreamSource(); s.setAlias(sc.alias()); s.setSqlQuery(sc.query()); if (sc.slide().isDefined()) s.setRawSlideValue(sc.slide().get()); if (sc.samplingRate().isDefined()) s.setSamplingRate(((Double)sc.samplingRate().get()).floatValue()); if (sc.disconnectBufferSize().isDefined()) s.setDisconnectedBufferSize(((Integer)sc.disconnectBufferSize().get())); if (sc.storageSize().isDefined()) s.setRawHistorySize(sc.storageSize().get()); AddressBean[] add=new AddressBean[sc.wrappers().size()]; int i=0; for (WrapperConf w:JavaConversions.asJavaIterable(sc.wrappers())){ add[i]=address(w); i++; } s.setAddressing(add); return s; }
@Override public String toString() { return "MesosConfiguration{" + "masterUrl='" + masterUrl + '\'' + ", frameworkInfo=" + frameworkInfo + ", credential=" + (credential.isDefined() ? "(not shown)" : "(none)") + '}'; }
/** * We retrieve the body apart from the request. Otherwise, there is an issue in casting the body between Scala and Java. * * @param requestHeader the request without the body * @param body the body (maybe) * @param sessionStore the session store */ public PlayWebContext(final RequestHeader requestHeader, final Object body, final SessionStore<org.pac4j.play.PlayWebContext> sessionStore) { this(JavaHelpers$.MODULE$.createJavaContext(requestHeader, JavaHelpers$.MODULE$.createContextComponents()), sessionStore); this.formParameters = new HashMap<>(); if (body instanceof AnyContentAsFormUrlEncoded) { final scala.collection.immutable.Map<String, Seq<String>> parameters = ((AnyContentAsFormUrlEncoded) body).asFormUrlEncoded().get(); for (final String key : ScalaCompatibility.scalaSetToJavaSet(parameters.keySet())) { final Seq<String> v = parameters.get(key).get(); final String[] values = new String[v.size()]; v.copyToArray(values); formParameters.put(key, values); } } }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { Iterator<AccumulatorV2<?, ?>> iterator = taskEnd.taskMetrics().accumulators().iterator(); while (iterator.hasNext()) { AccumulatorV2 accumulator = iterator.next(); if (taskEnd.stageId() == 1 && accumulator.isRegistered() && accumulator.name().isDefined() && accumulator.name().get().equals("internal.metrics.shuffle.read.recordsRead")) { stageOneShuffleReadTaskRecordsCountMap.put(taskEnd.taskInfo().taskId(), (Long) accumulator.value()); } } } });
public static AuthKey authKeyFromHeaders(HttpRequest request) throws Exception { Option<String> header = request.header("Authorization").headOption(); if (header.isDefined() && header.get().startsWith("Basic ")) { String encoded = header.get().replaceFirst("Basic ", ""); String decoded = new String(Base64.getDecoder().decode(encoded), "UTF-8"); String[] userPass = decoded.split(":", 2); return AuthKey.with(userPass[0], userPass[1]); } return null; }
private void executeAndVerifyProposals(ZkUtils zkUtils, Collection<ExecutionProposal> proposalsToExecute, Collection<ExecutionProposal> proposalsToCheck) { KafkaCruiseControlConfig configs = new KafkaCruiseControlConfig(getExecutorProperties()); Executor executor = new Executor(configs, new SystemTime(), new MetricRegistry(), 86400000L, 43200000L); executor.setExecutionMode(false); executor.executeProposals(proposalsToExecute, Collections.emptySet(), null, EasyMock.mock(LoadMonitor.class), null, null, null); Map<TopicPartition, Integer> replicationFactors = new HashMap<>(); for (ExecutionProposal proposal : proposalsToCheck) { int replicationFactor = zkUtils.getReplicasForPartition(proposal.topic(), proposal.partitionId()).size(); replicationFactors.put(new TopicPartition(proposal.topic(), proposal.partitionId()), replicationFactor); } waitUntilExecutionFinishes(executor); for (ExecutionProposal proposal : proposalsToCheck) { TopicPartition tp = new TopicPartition(proposal.topic(), proposal.partitionId()); int expectedReplicationFactor = replicationFactors.get(tp); assertEquals("Replication factor for partition " + tp + " should be " + expectedReplicationFactor, expectedReplicationFactor, zkUtils.getReplicasForPartition(tp.topic(), tp.partition()).size()); if (proposal.hasReplicaAction()) { for (int brokerId : proposal.newReplicas()) { assertTrue("The partition should have moved for " + tp, zkUtils.getReplicasForPartition(tp.topic(), tp.partition()).contains(brokerId)); } } assertEquals("The leader should have moved for " + tp, proposal.newLeader(), zkUtils.getLeaderForPartition(tp.topic(), tp.partition()).get()); } }