How to use

VertexProgram

in

org.apache.tinkerpop.gremlin.process.computer

public FulgoraMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers) {
  this.currentMap = new ConcurrentHashMap<>();
  this.previousMap = new ConcurrentHashMap<>();
  if (null != vertexProgram) {
    for (final String key : vertexProgram.getMemoryComputeKeys()) {
      MemoryHelper.validateKey(key);
      this.memoryKeys.add(key);
    }
  }
  for (final MapReduce mapReduce : mapReducers) {
    this.memoryKeys.add(mapReduce.getMemoryKey());
  }
}

@Override
public void workerIterationEnd(ScanMetrics metrics) {
  vertexProgram.workerIterationEnd(memory.asImmutable());
}

@Override
public void workerIterationStart(TitanGraph graph, Configuration config, ScanMetrics metrics) {
  vertexProgram.workerIterationStart(memory.asImmutable());
}

@Override
public RecordWriter<NullWritable, VertexWritable> getRecordWriter(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
  synchronized (this) {
    if (null == graph) {
      Configuration hadoopConf = taskAttemptContext.getConfiguration();
      ModifiableHadoopConfiguration mhc =
          ModifiableHadoopConfiguration.of(TitanHadoopConfiguration.MAPRED_NS, hadoopConf);
      graph = (StandardTitanGraph) TitanFactory.open(mhc.getTitanGraphConf());
    }
  }
  // Special case for a TP3 vertex program: persist only those properties whose keys are
  // returned by VertexProgram.getComputeKeys()
  if (null == persistableKeys) {
    try {
      persistableKeys = VertexProgram.createVertexProgram(graph,
          ConfUtil.makeApacheConfiguration(taskAttemptContext.getConfiguration())).getElementComputeKeys();
      log.debug("Set persistableKeys={}", Joiner.on(",").join(persistableKeys));
    } catch (Exception e) {
      log.debug("Unable to detect or instantiate vertex program", e);
      persistableKeys = ImmutableSet.of();
    }
  }
  StandardTitanTx tx = transactions.computeIfAbsent(taskAttemptContext.getTaskAttemptID(),
      id -> (StandardTitanTx)graph.newTransaction());
  return new TitanH1RecordWriter(taskAttemptContext, tx, persistableKeys);
}

  this.mapReduces.addAll(this.vertexProgram.getMapReducers());
this.persistMode = GraphComputerHelper.getPersistState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.persistMode));
this.resultGraphMode = GraphComputerHelper.getResultGraphState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.resultGraphMode));
    vertexProgram.setup(memory);
    memory.completeSubRound();
      vertexMemory.nextIteration(vertexProgram.getMessageScopes(memory));
      memory.completeSubRound();
      try {
        if (this.vertexProgram.terminate(this.memory)) {
          break;
  if (persistMode == Persist.NOTHING && resultGraphMode == ResultGraph.NEW) {
    resultgraph = EmptyGraph.instance();
  } else if (persistMode != Persist.NOTHING && vertexProgram != null && !vertexProgram.getElementComputeKeys().isEmpty()) {
      for (String key : vertexProgram.getElementComputeKeys()) {
        if (!mgmt.containsPropertyKey(key))
          log.warn("Property key [{}] is not part of the schema and will be created. It is advised to initialize all keys.", key);
      AtomicInteger failures = new AtomicInteger(0);
      try (WorkerPool workers = new WorkerPool(numThreads)) {
        List<Map.Entry<Long, Map<String, Object>>> subset = new ArrayList<>(writeBatchSize / vertexProgram.getElementComputeKeys().size());
        int currentSize = 0;
        for (Map.Entry<Long, Map<String, Object>> entry : mutatedProperties.entrySet()) {

private ComputerGraph(final State state, final Vertex starVertex, final Optional<VertexProgram<?>> vertexProgram) {
  this.state = state;
  this.computeKeys = vertexProgram.isPresent() ? vertexProgram.get().getVertexComputeKeys().stream().map(VertexComputeKey::getKey).collect(Collectors.toSet()) : Collections.emptySet();
  this.starVertex = new ComputerVertex(starVertex);
}

        return Collections.emptyIterator();
      final VertexProgram<M> workerVertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(graphComputerConfiguration), vertexProgramConfiguration); // each partition(Spark)/worker(TP3) has a local copy of the vertex program (a worker's task)
      final String[] vertexComputeKeysArray = VertexProgramHelper.vertexComputeKeysAsArray(workerVertexProgram.getVertexComputeKeys()); // the compute keys as an array
      final SparkMessenger<M> messenger = new SparkMessenger<>();
      workerVertexProgram.workerIterationStart(memory.asImmutable()); // start the worker
      return IteratorUtils.map(partitionIterator, vertexViewIncoming -> {
        final StarGraph.StarVertex vertex = vertexViewIncoming._2()._1().get(); // get the vertex from the vertex writable
        workerVertexProgram.execute(ComputerGraph.vertexProgram(vertex, workerVertexProgram), messenger, memory); // execute the vertex program on this vertex for this iteration
        final List<Tuple2<Object, M>> outgoingMessages = messenger.getOutgoingMessages(); // get the outgoing messages being sent by this vertex
        if (!partitionIterator.hasNext())
          workerVertexProgram.workerIterationEnd(memory.asImmutable()); // if no more vertices in the partition, end the worker's iteration
        return (nextView.isEmpty() && outgoingMessages.isEmpty()) ?
        IteratorUtils.of(new Tuple2<>(tuple._1(), tuple._2().getView())),      // emit the view payload
        IteratorUtils.map(tuple._2().getOutgoingMessages().iterator(), message -> new Tuple2<>(message._1(), new MessagePayload<>(message._2()))));
final MessageCombiner<M> messageCombiner = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(vertexProgramConfiguration), vertexProgramConfiguration).getMessageCombiner().orElse(null);
final Function2<Payload, Payload, Payload> reducerFunction = (a, b) -> {      // reduce the view and outgoing messages into a single payload object representing the new view and incoming messages for a vertex
  if (a instanceof ViewIncomingPayload) {

String inputLocation = Constants
    .getSearchGraphLocation(hadoopConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION),
        fileSystemStorage).orElse(null);
if (null != inputLocation) {
  try {
    this.vertexProgram.storeState(vertexProgramConfiguration);
    this.vertexProgram.setup(memory);
    JavaPairRDD<Object, ViewIncomingPayload<Object>> viewIncomingRDD = null;
    memory.broadcastMemory(sparkContext);
      viewIncomingRDD = SparkExecutor.executeVertexProgramIteration(loadedGraphRDD, viewIncomingRDD, memory, graphComputerConfiguration, vertexProgramConfiguration);
      memory.setInExecute(false);
      if (this.vertexProgram.terminate(memory))
        break;
      else {
      computedGraphRDD = SparkExecutor.prepareFinalGraphRDD(loadedGraphRDD, viewIncomingRDD, this.vertexProgram.getVertexComputeKeys());
      assert null != computedGraphRDD && computedGraphRDD != loadedGraphRDD;
    } else {

public static<M> MessageCombiner<M> getMessageCombiner(VertexProgram<M> program) {
  return program.getMessageCombiner().orElse(DEFAULT_COMBINER);
}

  this.mapReducers.addAll(this.vertexProgram.getMapReducers());
final Future<ComputerResult> result = computerService.submit(() -> {
  final long time = System.currentTimeMillis();
  final TinkerGraphComputerView view = TinkerHelper.createGraphComputerView(this.graph, this.graphFilter, null != this.vertexProgram ? this.vertexProgram.getVertexComputeKeys() : Collections.emptySet());
  final TinkerWorkerPool workers = new TinkerWorkerPool(this.graph, this.memory, this.workers);
  try {
    if (null != this.vertexProgram) {
      this.vertexProgram.setup(this.memory);
      while (true) {
        if (Thread.interrupted()) throw new TraversalInterruptedException();
        workers.setVertexProgram(this.vertexProgram);
        workers.executeVertexProgram((vertices, vertexProgram, workerMemory) -> {
          vertexProgram.workerIterationStart(workerMemory.asImmutable());
          while (vertices.hasNext()) {
            final Vertex vertex = vertices.next();
            if (Thread.interrupted()) throw new TraversalInterruptedException();
            vertexProgram.execute(
                ComputerGraph.vertexProgram(vertex, vertexProgram),
                new TinkerMessenger<>(vertex, this.messageBoard, vertexProgram.getMessageCombiner()),
                workerMemory);
          vertexProgram.workerIterationEnd(workerMemory.asImmutable());
          workerMemory.complete();
        });
        this.messageBoard.completeIteration();
        this.memory.completeSubRound();

protected void validateStatePriorToExecution() {
  // a graph computer can only be executed one time
  if (this.executed)
    throw Exceptions.computerHasAlreadyBeenSubmittedAVertexProgram();
  else
    this.executed = true;
  // it is not possible execute a computer if it has no vertex program nor mapreducers
  if (null == this.vertexProgram && this.mapReducers.isEmpty())
    throw GraphComputer.Exceptions.computerHasNoVertexProgramNorMapReducers();
  // it is possible to run mapreducers without a vertex program
  if (null != this.vertexProgram) {
    GraphComputerHelper.validateProgramOnComputer(this, vertexProgram);
    this.mapReducers.addAll(this.vertexProgram.getMapReducers());
  }
  // if the user didn't set desired persistence/resultgraph, then get from vertex program or else, no persistence
  this.persist = GraphComputerHelper.getPersistState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.persist));
  this.resultGraph = GraphComputerHelper.getResultGraphState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.resultGraph));
  // determine persistence and result graph options
  if (!this.features().supportsResultGraphPersistCombination(this.resultGraph, this.persist))
    throw GraphComputer.Exceptions.resultGraphPersistCombinationNotSupported(this.resultGraph, this.persist);
  // if too many workers are requested, throw appropriate exception
  if (this.workers > this.features().getMaxWorkers())
    throw GraphComputer.Exceptions.computerRequiresMoreWorkersThanSupported(this.workers, this.features().getMaxWorkers());
}

  this.mapReduces.addAll(this.vertexProgram.getMapReducers());
this.persistMode = GraphComputerHelper.getPersistState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.persistMode));
this.resultGraphMode = GraphComputerHelper.getResultGraphState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.resultGraphMode));
    vertexProgram.setup(memory);
        vertexMemory.nextIteration(vertexProgram.getMessageScopes(memory));
        memory.completeSubRound();
        try {
          if (this.vertexProgram.terminate(this.memory)) {
            break;
  if (persistMode == Persist.NOTHING && resultGraphMode == ResultGraph.NEW) {
    resultgraph = EmptyGraph.instance();
  } else if (persistMode != Persist.NOTHING && vertexProgram != null && !vertexProgram.getVertexComputeKeys().isEmpty()) {
      for (VertexComputeKey key : vertexProgram.getVertexComputeKeys()) {
        if (!management.containsPropertyKey(key.getKey()))
          log.warn("Property key [{}] is not part of the schema and will be created. It is advised to initialize all keys.", key.getKey());
      AtomicInteger failures = new AtomicInteger(0);
      try (WorkerPool workers = new WorkerPool(numThreads)) {
        List<Map.Entry<Long, Map<String, Object>>> subset = new ArrayList<>(writeBatchSize / vertexProgram.getVertexComputeKeys().size());
        int currentSize = 0;
        for (Map.Entry<Long, Map<String, Object>> entry : mutatedProperties.entrySet()) {

  @Override
  public <P extends VertexProgram> P create(final Graph graph) {
    return (P) VertexProgram.createVertexProgram(graph, this.configuration);
  }
}

public static void validateProgramOnComputer(final GraphComputer computer, final VertexProgram vertexProgram) {
  if (vertexProgram.getMemoryComputeKeys().contains(null))
    throw Memory.Exceptions.memoryKeyCanNotBeNull();
  if (vertexProgram.getMemoryComputeKeys().contains(""))
    throw Memory.Exceptions.memoryKeyCanNotBeEmpty();
  final GraphComputer.Features graphComputerFeatures = computer.features();
  final VertexProgram.Features vertexProgramFeatures = vertexProgram.getFeatures();
  for (final Method method : VertexProgram.Features.class.getMethods()) {
    if (method.getName().startsWith("requires")) {
      final boolean supports;
      final boolean requires;
      try {
        supports = (boolean) GraphComputer.Features.class.getMethod(method.getName().replace("requires", "supports")).invoke(graphComputerFeatures);
        requires = (boolean) method.invoke(vertexProgramFeatures);
      } catch (final Exception e) {
        throw new IllegalStateException("A reflection exception has occurred: " + e.getMessage(), e);
      }
      if (requires && !supports)
        throw new IllegalStateException("The vertex program can not be executed on the graph computer: " + method.getName());
    }
  }
}

@Override
public void process(TitanVertex vertex, ScanMetrics metrics) {
  PreloadedVertex v = (PreloadedVertex)vertex;
  long vertexId = v.longId();
  VertexMemoryHandler<M> vh = new VertexMemoryHandler(vertexMemory,v);
  v.setAccessCheck(PreloadedVertex.OPENSTAR_CHECK);
  if (idManager.isPartitionedVertex(vertexId)) {
    if (idManager.isCanonicalVertexId(vertexId)) {
      EntryList results = v.getFromCache(SYSTEM_PROPS_QUERY);
      if (results == null) results = EntryList.EMPTY_LIST;
      vertexMemory.setLoadedProperties(vertexId,results);
    }
    for (MessageScope scope : vertexMemory.getPreviousScopes()) {
      if (scope instanceof MessageScope.Local) {
        M combinedMsg = null;
        for (Iterator<M> msgIter = vh.receiveMessages(scope).iterator(); msgIter.hasNext(); ) {
          M msg = msgIter.next();
          if (combinedMsg==null) combinedMsg=msg;
          else combinedMsg = combiner.combine(combinedMsg,msg);
        }
        if (combinedMsg!=null) vertexMemory.aggregateMessage(vertexId,combinedMsg,scope);
      }
    }
  } else {
    v.setPropertyMixing(vh);
    vertexProgram.execute(v, vh, memory);
  }
}

public FulgoraVertexMemory(int numVertices, final IDManager idManager, final VertexProgram<M> vertexProgram) {
  Preconditions.checkArgument(numVertices>=0 && vertexProgram!=null && idManager!=null);
  vertexStates = new NonBlockingHashMapLong<>(numVertices);
  partitionVertices = new NonBlockingHashMapLong<>(64);
  this.idManager = idManager;
  this.combiner = FulgoraUtil.getMessageCombiner(vertexProgram);
  this.computeKeys = vertexProgram.getVertexComputeKeys();
  this.elementKeyMap = getIdMap(vertexProgram.getVertexComputeKeys().stream().map(VertexComputeKey::getKey).collect(Collectors.toCollection(HashSet::new)));
  this.previousScopes = ImmutableMap.of();
}

private ComputerGraph(final State state, final Vertex starVertex, final Optional<VertexProgram<?>> vertexProgram) {
  this.state = state;
  this.computeKeys = vertexProgram.isPresent() ? vertexProgram.get().getVertexComputeKeys().stream().map(VertexComputeKey::getKey).collect(Collectors.toSet()) : Collections.emptySet();
  this.starVertex = new ComputerVertex(starVertex);
}

        return Collections.emptyIterator();}
      final VertexProgram<M> workerVertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(graphComputerConfiguration), vertexProgramConfiguration); // each partition(Spark)/worker(TP3) has a local copy of the vertex program (a worker's task)
      final String[] vertexComputeKeysArray = VertexProgramHelper.vertexComputeKeysAsArray(workerVertexProgram.getVertexComputeKeys()); // the compute keys as an array
      final SparkMessenger<M> messenger = new SparkMessenger<>();
      workerVertexProgram.workerIterationStart(memory.asImmutable()); // start the worker
      return IteratorUtils.map(partitionIterator, vertexViewIncoming -> {
        final StarGraph.StarVertex vertex = vertexViewIncoming._2()._1().get(); // get the vertex from the vertex writable
        workerVertexProgram.execute(ComputerGraph.vertexProgram(vertex, workerVertexProgram), messenger, memory); // execute the vertex program on this vertex for this iteration
        final List<Tuple2<Object, M>> outgoingMessages = messenger.getOutgoingMessages(); // get the outgoing messages being sent by this vertex
        if (!partitionIterator.hasNext()) {
          workerVertexProgram.workerIterationEnd(memory.asImmutable()); // if no more vertices in the partition, end the worker's iteration}
        }return (nextView.isEmpty() && outgoingMessages.isEmpty()) ?
        IteratorUtils.of(new Tuple2<>(tuple._1(), tuple._2().getView())),      // emit the view payload
        IteratorUtils.map(tuple._2().getOutgoingMessages().iterator(), message -> new Tuple2<>(message._1(), new MessagePayload<>(message._2()))));
final MessageCombiner<M> messageCombiner = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(vertexProgramConfiguration), vertexProgramConfiguration).getMessageCombiner().orElse(null);
final Function2<Payload, Payload, Payload> reducerFunction = (a, b) -> {      // reduce the view and outgoing messages into a single payload object representing the new view and incoming messages for a vertex
  if (a instanceof ViewIncomingPayload) {

String inputLocation = Constants
    .getSearchGraphLocation(hadoopConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION),
        fileSystemStorage).orElse(null);
if (null != inputLocation) {
  try {
    this.vertexProgram.storeState(vertexProgramConfiguration);
    this.vertexProgram.setup(memory);
    JavaPairRDD<Object, ViewIncomingPayload<Object>> viewIncomingRDD = null;
    memory.broadcastMemory(sparkContext);
      viewIncomingRDD = SparkExecutor.executeVertexProgramIteration(loadedGraphRDD, viewIncomingRDD, memory, graphComputerConfiguration, vertexProgramConfiguration);
      memory.setInExecute(false);
      if (this.vertexProgram.terminate(memory))
        break;
      else {
      computedGraphRDD = SparkExecutor.prepareFinalGraphRDD(loadedGraphRDD, viewIncomingRDD, this.vertexProgram.getVertexComputeKeys());
      assert null != computedGraphRDD && computedGraphRDD != loadedGraphRDD;
    } else {

public static<M> MessageCombiner<M> getMessageCombiner(VertexProgram<M> program) {
  return program.getMessageCombiner().orElse(DEFAULT_COMBINER);
}