private Builder(final EvaluatorRequest request) { setNumber(request.getNumber()); fromDescriptor(request.getDescriptor()); }
/** * @return a new EvaluatorRequest Builder. */ public static Builder newBuilder() { return new Builder(); }
/** * Builds the {@link EvaluatorRequest}. */ @Override public EvaluatorRequest build() { return new EvaluatorRequest(this.n, this.megaBytes, this.cores, this.descriptor); }
@Override public void onNext(final FailedEvaluator failedEvaluator) { final String evalId = failedEvaluator.getId(); final Configuration computeConfig = submittedComputeEvalConfigs.remove(evalId); if (computeConfig != null) { LOG.log(Level.INFO, "Received failed compute evaluator: {0}", evalId); failedComputeEvalConfigs.add(computeConfig); requestor.submit(EvaluatorRequest.newBuilder() .setMemory(computeEvalMemoryMB).setNumber(1).setNumberOfCores(computeEvalCore).build()); } else { final Pair<Configuration, Configuration> confPair = submittedDataEvalConfigs.remove(evalId); if (confPair != null) { LOG.log(Level.INFO, "Received failed data evaluator: {0}", evalId); failedDataEvalConfigs.add(confPair); requestor.submit(EvaluatorRequest.newBuilder() .setMemory(dataEvalMemoryMB).setNumber(1).setNumberOfCores(dataEvalCore).build()); } else { LOG.log(Level.SEVERE, "Received unknown failed evaluator " + evalId, failedEvaluator.getEvaluatorException()); throw new RuntimeException("Received failed evaluator that I did not submit: " + evalId); } } } }
private EvaluatorRequest getDataLoadingRequest() { return EvaluatorRequest.newBuilder() .setNumber(this.dataLoadingService.getNumberOfPartitions()) .setMemory(this.dataEvalMemoryMB) .setNumberOfCores(this.dataEvalCore) .build(); }
@Override public Configuration getContextConfiguration(final AllocatedEvaluator allocatedEvaluator) { final NumberedSplit<InputSplit> numberedSplit = this.evaluatorToPartitionMapper.getInputSplit( allocatedEvaluator.getEvaluatorDescriptor().getNodeDescriptor().getName(), allocatedEvaluator.getId()); return ContextConfiguration.CONF .set(ContextConfiguration.IDENTIFIER, DATA_LOAD_CONTEXT_PREFIX + numberedSplit.getIndex()) .build(); }
@Override public synchronized void submit(final EvaluatorRequest req) { LOG.log(Level.FINEST, "Got an EvaluatorRequest: number: {0}, memory = {1}, cores = {2}.", new Object[] {req.getNumber(), req.getMegaBytes(), req.getNumberOfCores() }); if (req.getMegaBytes() <= 0) { throw new IllegalArgumentException("Given an unsupported memory size: " + req.getMegaBytes()); if (req.getNumberOfCores() <= 0) { throw new IllegalArgumentException("Given an unsupported core number: " + req.getNumberOfCores()); if (req.getNumber() <= 0) { throw new IllegalArgumentException("Given an unsupported number of evaluators: " + req.getNumber()); .setResourceCount(req.getNumber()) .setVirtualCores(req.getNumberOfCores()) .setMemorySize(req.getMegaBytes()); final ResourceCatalog.Descriptor descriptor = req.getDescriptor(); if (descriptor != null) { if (descriptor instanceof RackDescriptor) {
public static String serialize(EvaluatorRequest request){ try(ByteArrayOutputStream baos = new ByteArrayOutputStream()){ try(DataOutputStream daos = new DataOutputStream(baos)){ daos.writeInt(request.getNumber()); daos.writeInt(request.getMegaBytes()); daos.writeInt(request.getNumberOfCores()); } catch (IOException e) { throw e; } return Base64.encodeBase64String(baos.toByteArray()); } catch (IOException e1) { throw new RuntimeException("Unable to serialize compute request", e1); } }
final String nodeId = evaluatorDescriptor.getNodeDescriptor().getId(); final String nodeName = evaluatorDescriptor.getNodeDescriptor().getName(); final InetSocketAddress address = evaluatorDescriptor.getNodeDescriptor().getInetSocketAddress(); writer.println("Evaluator InternetAddress: " + address); writer.write("<br/>"); writer.println("Evaluator Memory: " + evaluatorDescriptor.getMemory()); writer.write("<br/>"); writer.println("Evaluator Core: " + evaluatorDescriptor.getNumberOfCores()); writer.write("<br/>"); writer.println("Evaluator Type: " + evaluatorDescriptor.getType()); writer.write("<br/>"); } else {
nodeId = evaluatorDescriptor.getNodeDescriptor().getId(); nodeName = evaluatorDescriptor.getNodeDescriptor().getName(); address = evaluatorDescriptor.getNodeDescriptor().getInetSocketAddress(); memory = evaluatorDescriptor.getMemory(); type = evaluatorDescriptor.getType().toString();
@Override public void onNext(final AllocatedEvaluator eval) { synchronized (ReefEventStateManager.this) { ReefEventStateManager.this.put(eval.getId(), eval.getEvaluatorDescriptor()); } } }
@Override public void onNext(final EvaluatorRequest request) { try { LOG.log(Level.FINE, "Processing a request with count: {0} - Waiting for gate to be released", request.getNumber()); this.resourceRequestGate.await(); LOG.log(Level.FINE, "Gate released. Submitting request: {0}", request); this.resourceRequestGate = new CountDownLatch(1); this.requestor.submit(request); } catch (final InterruptedException ex) { LOG.log(Level.FINEST, "Interrupted", ex); } } }
/** * get Evaluator NodeDescriptor * * @param evaluatorId * @return */ public NodeDescriptor getEvaluatorNodeDescriptor(final String evaluatorId) { return evaluators.get(evaluatorId).getNodeDescriptor(); }
@Override public void onNext(final FailedEvaluator failedEvaluator) { throw new RuntimeException( "No handler bound for FailedEvaluator: " + failedEvaluator, failedEvaluator.getEvaluatorException()); } }
@Override public void onNext(final AllocatedEvaluator allocatedEvaluator) { LOG.log(Level.INFO, "Received AllocatedEvaluator: {0} :: CLOSING", allocatedEvaluator); allocatedEvaluator.close(); } }
public static EvaluatorRequest deserialize(String serializedRequest){ try(ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(serializedRequest))){ try(DataInputStream dais = new DataInputStream(bais)){ return EvaluatorRequest.newBuilder() .setNumber(dais.readInt()) .setMemory(dais.readInt()) .setNumberOfCores(dais.readInt()) .build(); } } catch (IOException e) { throw new RuntimeException("Unable to de-serialize compute request", e); } } }
@Override public Configuration getServiceConfiguration(final AllocatedEvaluator allocatedEvaluator) { try { final NumberedSplit<InputSplit> numberedSplit = this.evaluatorToPartitionMapper.getInputSplit( allocatedEvaluator.getEvaluatorDescriptor().getNodeDescriptor().getName(), allocatedEvaluator.getId()); final Configuration serviceConfiguration = ServiceConfiguration.CONF .set(ServiceConfiguration.SERVICES, this.inMemory ? InMemoryInputFormatDataSet.class : InputFormatDataSet.class) .build(); return Tang.Factory.getTang().newConfigurationBuilder(serviceConfiguration) .bindImplementation( DataSet.class, this.inMemory ? InMemoryInputFormatDataSet.class : InputFormatDataSet.class) .bindNamedParameter(JobConfExternalConstructor.InputFormatClass.class, inputFormatClass) .bindNamedParameter(JobConfExternalConstructor.InputPath.class, inputPath) .bindNamedParameter( InputSplitExternalConstructor.SerializedInputSplit.class, WritableSerializer.serialize(numberedSplit.getEntry())) .bindConstructor(InputSplit.class, InputSplitExternalConstructor.class) .bindConstructor(JobConf.class, JobConfExternalConstructor.class) .build(); } catch (final BindException ex) { final String evalId = allocatedEvaluator.getId(); final String msg = "Unable to create configuration for evaluator " + evalId; LOG.log(Level.WARNING, msg, ex); throw new RuntimeException(msg, ex); } }
/** * Get all evaluator ids and send it back to response so that can be displayed on web * * @param response * @throws IOException */ private void writeEvaluatorsWebOutput(final HttpServletResponse response) throws IOException { LOG.log(Level.INFO, "HttpServerReefEventHandler getEvaluators is called"); final PrintWriter writer = response.getWriter(); writer.println("<h1>Evaluators:</h1>"); for (final Map.Entry<String, EvaluatorDescriptor> entry : this.reefStateManager.getEvaluators().entrySet()) { final String key = entry.getKey(); final EvaluatorDescriptor descriptor = entry.getValue(); writer.println("Evaluator Id: " + key); writer.write("<br/>"); writer.println("Evaluator Name: " + descriptor.getNodeDescriptor().getName()); writer.write("<br/>"); } writer.write("<br/>"); writer.println("Total number of Evaluators: " + this.reefStateManager.getEvaluators().size()); writer.write("<br/>"); writer.println(String.format("Driver Start Time:[%s]", this.reefStateManager.getStartTime())); }
/** * @return a new EvaluatorRequest Builder with settings initialized * from an existing request. */ public static Builder newBuilder(final EvaluatorRequest request) { return new Builder(request); }
/** * Build AvroEvaluatorList object. */ @Override public AvroEvaluatorList toAvro( final Map<String, EvaluatorDescriptor> evaluatorMap, final int totalEvaluators, final String startTime) { final List<AvroEvaluatorEntry> EvaluatorEntities = new ArrayList<>(); for (final Map.Entry<String, EvaluatorDescriptor> entry : evaluatorMap.entrySet()) { final EvaluatorDescriptor descriptor = entry.getValue(); EvaluatorEntities.add(AvroEvaluatorEntry.newBuilder() .setId(entry.getKey()) .setName(descriptor.getNodeDescriptor().getName()) .build()); } return AvroEvaluatorList.newBuilder() .setEvaluators(EvaluatorEntities) .setTotal(totalEvaluators) .setStartTime(startTime != null ? startTime : new Date().toString()) .build(); }