@Override public void initialize(final InputSplit inputSplit, final TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { final Configuration configuration = taskAttemptContext.getConfiguration(); final InputFormat<NullWritable, VertexWritable> inputFormat = ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_READER, InputFormat.class, InputFormat.class), configuration); if (!(inputFormat instanceof GraphFilterAware) && configuration.get(Constants.GREMLIN_HADOOP_GRAPH_FILTER, null) != null) this.graphFilter = VertexProgramHelper.deserialize(ConfUtil.makeApacheConfiguration(configuration), Constants.GREMLIN_HADOOP_GRAPH_FILTER); this.recordReader = inputFormat.createRecordReader(inputSplit, taskAttemptContext); this.recordReader.initialize(inputSplit, taskAttemptContext); }
@Override public void initialize(final InputSplit genericSplit, final TaskAttemptContext context) throws IOException { this.lineRecordReader.initialize(genericSplit, context); final Configuration configuration = context.getConfiguration(); if (configuration.get(Constants.GREMLIN_HADOOP_GRAPH_FILTER, null) != null) this.graphFilter = VertexProgramHelper.deserialize(ConfUtil.makeApacheConfiguration(configuration), Constants.GREMLIN_HADOOP_GRAPH_FILTER); this.engine = manager.getEngineByName(configuration.get(SCRIPT_ENGINE, "gremlin-groovy")); final FileSystem fs = FileSystem.get(configuration); try (final InputStream stream = fs.open(new Path(configuration.get(SCRIPT_FILE))); final InputStreamReader reader = new InputStreamReader(stream)) { final String parse = String.join("\n", IOUtils.toString(reader), READ_CALL); script = ((Compilable) engine).compile(parse); } catch (ScriptException e) { throw new IOException(e.getMessage()); } }
public static <S, E> PureTraversal<S, E> loadState(final Configuration configuration, final String configurationKey, final Graph graph) { final Object configValue = configuration.getProperty(configurationKey); final PureTraversal<S, E> pureTraversal = (configValue instanceof String ? (PureTraversal<S, E>) VertexProgramHelper.deserialize(configuration, configurationKey) : ((PureTraversal<S, E>) configValue)); pureTraversal.pureTraversal.setGraph(graph); return pureTraversal; }
@Override public void initialize(final InputSplit genericSplit, final TaskAttemptContext context) throws IOException { final FileSplit split = (FileSplit) genericSplit; final Configuration configuration = context.getConfiguration(); if (configuration.get(Constants.GREMLIN_HADOOP_GRAPH_FILTER, null) != null) this.graphFilter = VertexProgramHelper.deserialize(ConfUtil.makeApacheConfiguration(configuration), Constants.GREMLIN_HADOOP_GRAPH_FILTER); this.gryoReader = GryoReader.build().mapper( GryoMapper.build().addRegistries(IoRegistryHelper.createRegistries(ConfUtil.makeApacheConfiguration(configuration))).create()).create(); long start = split.getStart(); final Path file = split.getPath(); if (null != new CompressionCodecFactory(configuration).getCodec(file)) { throw new IllegalStateException("Compression is not supported for the (binary) Gryo format"); } // open the file and seek to the start of the split this.inputStream = file.getFileSystem(configuration).open(split.getPath()); this.splitLength = split.getLength(); if (this.splitLength > 0) this.splitLength -= (seekToHeader(this.inputStream, start) - start); }
public static <R> TraverserSet<R> loadHaltedTraversers(final Configuration configuration) { if (!configuration.containsKey(HALTED_TRAVERSERS)) return new TraverserSet<>(); final Object object = configuration.getProperty(HALTED_TRAVERSERS) instanceof String ? VertexProgramHelper.deserialize(configuration, HALTED_TRAVERSERS) : configuration.getProperty(HALTED_TRAVERSERS); if (object instanceof Traverser.Admin) return new TraverserSet<>((Traverser.Admin<R>) object); else { final TraverserSet<R> traverserSet = new TraverserSet<>(); traverserSet.addAll((Collection) object); return traverserSet; } }
@Override public void initialize(final InputSplit inputSplit, final TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { reader.initialize(inputSplit, taskAttemptContext); final Configuration conf = taskAttemptContext.getConfiguration(); if (conf.get(Constants.GREMLIN_HADOOP_GRAPH_FILTER, null) != null) { graphFilter = VertexProgramHelper.deserialize(ConfUtil.makeApacheConfiguration(conf), Constants.GREMLIN_HADOOP_GRAPH_FILTER); } }
@Override public void initialize(final InputSplit inputSplit, final TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { reader.initialize(inputSplit, taskAttemptContext); final Configuration conf = taskAttemptContext.getConfiguration(); if (conf.get(Constants.GREMLIN_HADOOP_GRAPH_FILTER, null) != null) { graphFilter = VertexProgramHelper.deserialize(ConfUtil.makeApacheConfiguration(conf), Constants.GREMLIN_HADOOP_GRAPH_FILTER); } }
@Override public void initialize(final InputSplit inputSplit, final TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { final Configuration configuration = taskAttemptContext.getConfiguration(); final InputFormat<NullWritable, VertexWritable> inputFormat = ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_READER, InputFormat.class, InputFormat.class), configuration); if (!(inputFormat instanceof GraphFilterAware) && configuration.get(Constants.GREMLIN_HADOOP_GRAPH_FILTER, null) != null) this.graphFilter = VertexProgramHelper.deserialize(ConfUtil.makeApacheConfiguration(configuration), Constants.GREMLIN_HADOOP_GRAPH_FILTER); this.recordReader = inputFormat.createRecordReader(inputSplit, taskAttemptContext); this.recordReader.initialize(inputSplit, taskAttemptContext); }
public static <S, E> PureTraversal<S, E> loadState(final Configuration configuration, final String configurationKey, final Graph graph) { final Object configValue = configuration.getProperty(configurationKey); final PureTraversal<S, E> pureTraversal = (configValue instanceof String ? (PureTraversal<S, E>) VertexProgramHelper.deserialize(configuration, configurationKey) : ((PureTraversal<S, E>) configValue)); pureTraversal.pureTraversal.setGraph(graph); return pureTraversal; }
@Override public void initialize(final InputSplit genericSplit, final TaskAttemptContext context) throws IOException { this.lineRecordReader.initialize(genericSplit, context); final Configuration configuration = context.getConfiguration(); if (configuration.get(Constants.GREMLIN_HADOOP_GRAPH_FILTER, null) != null) this.graphFilter = VertexProgramHelper.deserialize(ConfUtil.makeApacheConfiguration(configuration), Constants.GREMLIN_HADOOP_GRAPH_FILTER); this.engine = manager.getEngineByName(configuration.get(SCRIPT_ENGINE, "gremlin-groovy")); final FileSystem fs = FileSystem.get(configuration); try (final InputStream stream = fs.open(new Path(configuration.get(SCRIPT_FILE))); final InputStreamReader reader = new InputStreamReader(stream)) { final String parse = String.join("\n", IOUtils.toString(reader), READ_CALL); script = ((Compilable) engine).compile(parse); } catch (ScriptException e) { throw new IOException(e.getMessage()); } }
@Override public void initialize(final InputSplit genericSplit, final TaskAttemptContext context) throws IOException { final FileSplit split = (FileSplit) genericSplit; final Configuration configuration = context.getConfiguration(); if (configuration.get(Constants.GREMLIN_HADOOP_GRAPH_FILTER, null) != null) this.graphFilter = VertexProgramHelper.deserialize(ConfUtil.makeApacheConfiguration(configuration), Constants.GREMLIN_HADOOP_GRAPH_FILTER); this.gryoReader = GryoReader.build().mapper( GryoMapper.build().addRegistries(IoRegistryHelper.createRegistries(ConfUtil.makeApacheConfiguration(configuration))).create()).create(); long start = split.getStart(); final Path file = split.getPath(); if (null != new CompressionCodecFactory(configuration).getCodec(file)) { throw new IllegalStateException("Compression is not supported for the (binary) Gryo format"); } // open the file and seek to the start of the split this.inputStream = file.getFileSystem(configuration).open(split.getPath()); this.splitLength = split.getLength(); if (this.splitLength > 0) this.splitLength -= (seekToHeader(this.inputStream, start) - start); }
public static <R> TraverserSet<R> loadHaltedTraversers(final Configuration configuration) { if (!configuration.containsKey(HALTED_TRAVERSERS)) return new TraverserSet<>(); final Object object = configuration.getProperty(HALTED_TRAVERSERS) instanceof String ? VertexProgramHelper.deserialize(configuration, HALTED_TRAVERSERS) : configuration.getProperty(HALTED_TRAVERSERS); if (object instanceof Traverser.Admin) return new TraverserSet<>((Traverser.Admin<R>) object); else { final TraverserSet<R> traverserSet = new TraverserSet<>(); traverserSet.addAll((Collection) object); return traverserSet; } }