private void addCustomClasses(final Map<String, Object> config, final GryoMapper.Builder builder) { final List<String> classNameList = getListStringFromConfig(TOKEN_CUSTOM, config); classNameList.stream().forEach(serializerDefinition -> { String className; Optional<String> serializerName; if (serializerDefinition.contains(";")) { final String[] split = serializerDefinition.split(";"); if (split.length != 2) throw new IllegalStateException(String.format("Invalid format for serializer definition [%s] - expected <class>;<serializer-class>", serializerDefinition)); className = split[0]; serializerName = Optional.of(split[1]); } else { serializerName = Optional.empty(); className = serializerDefinition; } try { final Class clazz = Class.forName(className); final Serializer serializer; if (serializerName.isPresent()) { final Class serializerClazz = Class.forName(serializerName.get()); serializer = (Serializer) serializerClazz.newInstance(); builder.addCustom(clazz, kryo -> serializer); } else builder.addCustom(clazz); } catch (Exception ex) { throw new IllegalStateException("Class could not be found", ex); } }); }
public GryoSerializer(final SparkConf sparkConfiguration) { final long bufferSizeKb = sparkConfiguration.getSizeAsKb("spark.kryoserializer.buffer", "64k"); final long maxBufferSizeMb = sparkConfiguration.getSizeAsMb("spark.kryoserializer.buffer.max", "64m"); this.referenceTracking = sparkConfiguration.getBoolean("spark.kryo.referenceTracking", true); this.registrationRequired = sparkConfiguration.getBoolean(Constants.SPARK_KRYO_REGISTRATION_REQUIRED, false); if (bufferSizeKb >= ByteUnit.GiB.toKiB(2L)) { throw new IllegalArgumentException("spark.kryoserializer.buffer must be less than 2048 mb, got: " + bufferSizeKb + " mb."); } else { this.bufferSize = (int) ByteUnit.KiB.toBytes(bufferSizeKb); if (maxBufferSizeMb >= ByteUnit.GiB.toMiB(2L)) { throw new IllegalArgumentException("spark.kryoserializer.buffer.max must be less than 2048 mb, got: " + maxBufferSizeMb + " mb."); } else { this.maxBufferSize = (int) ByteUnit.MiB.toBytes(maxBufferSizeMb); //this.userRegistrator = sparkConfiguration.getOption("spark.kryo.registrator"); } } // create a GryoPool and store it in static HadoopPools final List<Object> ioRegistries = new ArrayList<>(); ioRegistries.addAll(makeApacheConfiguration(sparkConfiguration).getList(IoRegistry.IO_REGISTRY, Collections.emptyList())); ioRegistries.add(SparkIoRegistry.class.getCanonicalName().replace("." + SparkIoRegistry.class.getSimpleName(), "$" + SparkIoRegistry.class.getSimpleName())); HadoopPools.initialize(GryoPool.build(). version(GryoVersion.valueOf(sparkConfiguration.get(GryoPool.CONFIG_IO_GRYO_VERSION, GryoPool.CONFIG_IO_GRYO_POOL_VERSION_DEFAULT.name()))). poolSize(sparkConfiguration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT)). ioRegistries(ioRegistries). initializeMapper(builder -> builder.referenceTracking(this.referenceTracking). registrationRequired(this.registrationRequired)). create()); }
final GryoMapper.Builder builder = GryoMapper.build(); detectRegistries().forEach(builder::addRegistry); return GryoReader.build().mapper(builder.create()).create(); } else if (objectOrClass.equals(IO.graphml)) return GraphMLReader.build().create();
final GryoMapper.Builder builder = GryoMapper.build(); detectRegistries().forEach(builder::addRegistry); return GryoWriter.build().mapper(builder.create()).create(); }else if (objectOrClass.equals(IO.graphml)) return GraphMLWriter.build().create();
public GryoSerializer(final SparkConf sparkConfiguration) { final long bufferSizeKb = sparkConfiguration.getSizeAsKb("spark.kryoserializer.buffer", "64k"); final long maxBufferSizeMb = sparkConfiguration.getSizeAsMb("spark.kryoserializer.buffer.max", "64m"); this.referenceTracking = sparkConfiguration.getBoolean("spark.kryo.referenceTracking", true); this.registrationRequired = sparkConfiguration.getBoolean(Constants.SPARK_KRYO_REGISTRATION_REQUIRED, false); if (bufferSizeKb >= ByteUnit.GiB.toKiB(2L)) { throw new IllegalArgumentException("spark.kryoserializer.buffer must be less than 2048 mb, got: " + bufferSizeKb + " mb."); } else { this.bufferSize = (int) ByteUnit.KiB.toBytes(bufferSizeKb); if (maxBufferSizeMb >= ByteUnit.GiB.toMiB(2L)) { throw new IllegalArgumentException("spark.kryoserializer.buffer.max must be less than 2048 mb, got: " + maxBufferSizeMb + " mb."); } else { this.maxBufferSize = (int) ByteUnit.MiB.toBytes(maxBufferSizeMb); //this.userRegistrator = sparkConfiguration.getOption("spark.kryo.registrator"); } } // create a GryoPool and store it in static HadoopPools final List<Object> ioRegistries = new ArrayList<>(); ioRegistries.addAll(makeApacheConfiguration(sparkConfiguration).getList(IoRegistry.IO_REGISTRY, Collections.emptyList())); ioRegistries.add(SparkIoRegistry.class.getCanonicalName().replace("." + SparkIoRegistry.class.getSimpleName(), "$" + SparkIoRegistry.class.getSimpleName())); HadoopPools.initialize(GryoPool.build(). version(GryoVersion.valueOf(sparkConfiguration.get(GryoPool.CONFIG_IO_GRYO_VERSION, GryoPool.CONFIG_IO_GRYO_POOL_VERSION_DEFAULT.name()))). poolSize(sparkConfiguration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT)). ioRegistries(ioRegistries). initializeMapper(builder -> builder.referenceTracking(this.referenceTracking). registrationRequired(this.registrationRequired)). create()); }
private void addCustomClasses(final Map<String, Object> config, final GryoMapper.Builder builder) { final List<String> classNameList = getListStringFromConfig(TOKEN_CUSTOM, config); classNameList.stream().forEach(serializerDefinition -> { String className; Optional<String> serializerName; if (serializerDefinition.contains(";")) { final String[] split = serializerDefinition.split(";"); if (split.length != 2) throw new IllegalStateException(String.format("Invalid format for serializer definition [%s] - expected <class>;<serializer-class>", serializerDefinition)); className = split[0]; serializerName = Optional.of(split[1]); } else { serializerName = Optional.empty(); className = serializerDefinition; } try { final Class clazz = Class.forName(className); final Serializer serializer; if (serializerName.isPresent()) { final Class serializerClazz = Class.forName(serializerName.get()); serializer = (Serializer) serializerClazz.newInstance(); builder.addCustom(clazz, kryo -> serializer); } else builder.addCustom(clazz); } catch (Exception ex) { throw new IllegalStateException("Class could not be found", ex); } }); }
private void addCustomClasses(final Map<String, Object> config, final GryoMapper.Builder builder) { final List<String> classNameList = getListStringFromConfig(TOKEN_CUSTOM, config); classNameList.stream().forEach(serializerDefinition -> { String className; Optional<String> serializerName; if (serializerDefinition.contains(";")) { final String[] split = serializerDefinition.split(";"); if (split.length != 2) throw new IllegalStateException(String.format("Invalid format for serializer definition [%s] - expected <class>;<serializer-class>", serializerDefinition)); className = split[0]; serializerName = Optional.of(split[1]); } else { serializerName = Optional.empty(); className = serializerDefinition; } try { final Class clazz = Class.forName(className); final Serializer serializer; if (serializerName.isPresent()) { final Class serializerClazz = Class.forName(serializerName.get()); serializer = (Serializer) serializerClazz.newInstance(); builder.addCustom(clazz, kryo -> serializer); } else builder.addCustom(clazz); } catch (Exception ex) { throw new IllegalStateException("Class could not be found", ex); } }); }
/** * Creates a {@code GryoMapper}. */ public GryoMapper create() { // consult the registry if provided and inject registry entries as custom classes. registries.forEach(registry -> { final List<Pair<Class, Object>> serializers = registry.find(GryoIo.class); serializers.forEach(p -> { if (null == p.getValue1()) addCustom(p.getValue0()); else if (p.getValue1() instanceof SerializerShim) addCustom(p.getValue0(), new ShadedSerializerAdapter((SerializerShim) p.getValue1())); else if (p.getValue1() instanceof Serializer) addCustom(p.getValue0(), (Serializer) p.getValue1()); else if (p.getValue1() instanceof Function) addCustom(p.getValue0(), (Function<Kryo, Serializer>) p.getValue1()); else throw new IllegalStateException(String.format( "Unexpected value provided by %s for serializable class %s - expected a parameter in [null, %s implementation or Function<%s, %s>], but received %s", registry.getClass().getSimpleName(), p.getValue0().getClass().getCanonicalName(), Serializer.class.getName(), Kryo.class.getSimpleName(), Serializer.class.getSimpleName(), p.getValue1())); }); }); return new GryoMapper(this); }
@Parameterized.Parameters(name = "{0}") public static Iterable<Object[]> data() { final SimpleModule moduleV1d0 = new SimpleModule(); moduleV1d0.addSerializer(CustomId.class, new CustomId.CustomIdJacksonSerializerV1d0()); final SimpleModule moduleV2d0 = new CustomId.CustomIdTinkerPopJacksonModuleV2d0(); final SimpleModule modulev3d0 = new CustomId.CustomIdTinkerPopJacksonModuleV3d0(); return Arrays.asList(new Object[][]{ {"graphson-v1-embedded", true, (Function<Graph, GraphReader>) g -> g.io(GraphSONIo.build(GraphSONVersion.V1_0)).reader().mapper(g.io(GraphSONIo.build(GraphSONVersion.V1_0)).mapper().addCustomModule(moduleV1d0).typeInfo(TypeInfo.PARTIAL_TYPES).create()).create(), (Function<Graph, GraphWriter>) g -> g.io(GraphSONIo.build(GraphSONVersion.V1_0)).writer().mapper(g.io(GraphSONIo.build(GraphSONVersion.V1_0)).mapper().addCustomModule(moduleV1d0).typeInfo(TypeInfo.PARTIAL_TYPES).create()).create()}, {"graphson-v2-embedded", true, (Function<Graph, GraphReader>) g -> g.io(GraphSONIo.build(GraphSONVersion.V2_0)).reader().mapper(g.io(GraphSONIo.build(GraphSONVersion.V2_0)).mapper().addCustomModule(moduleV2d0).typeInfo(TypeInfo.PARTIAL_TYPES).create()).create(), (Function<Graph, GraphWriter>) g -> g.io(GraphSONIo.build(GraphSONVersion.V2_0)).writer().mapper(g.io(GraphSONIo.build(GraphSONVersion.V2_0)).mapper().addCustomModule(moduleV2d0).typeInfo(TypeInfo.PARTIAL_TYPES).create()).create()}, {"graphson-v3", true, (Function<Graph, GraphReader>) g -> g.io(GraphSONIo.build(GraphSONVersion.V3_0)).reader().mapper(g.io(GraphSONIo.build(GraphSONVersion.V3_0)).mapper().addCustomModule(modulev3d0).create()).create(), (Function<Graph, GraphWriter>) g -> g.io(GraphSONIo.build(GraphSONVersion.V3_0)).writer().mapper(g.io(GraphSONIo.build(GraphSONVersion.V3_0)).mapper().addCustomModule(modulev3d0).create()).create()}, {"gryo-v1", true, (Function<Graph, GraphReader>) g -> g.io(GryoIo.build(GryoVersion.V1_0)).reader().mapper(g.io(GryoIo.build(GryoVersion.V1_0)).mapper().version(GryoVersion.V1_0).addCustom(CustomId.class).create()).create(), (Function<Graph, GraphWriter>) g -> g.io(GryoIo.build(GryoVersion.V1_0)).writer().mapper(g.io(GryoIo.build(GryoVersion.V1_0)).mapper().version(GryoVersion.V1_0).addCustom(CustomId.class).create()).create()}, {"gryo-v3", true, (Function<Graph, GraphReader>) g -> g.io(GryoIo.build(GryoVersion.V3_0)).reader().mapper(g.io(GryoIo.build(GryoVersion.V3_0)).mapper().version(GryoVersion.V3_0).addCustom(CustomId.class).create()).create(), (Function<Graph, GraphWriter>) g -> g.io(GryoIo.build(GryoVersion.V3_0)).writer().mapper(g.io(GryoIo.build(GryoVersion.V3_0)).mapper().version(GryoVersion.V3_0).addCustom(CustomId.class).create()).create()} }); }
@Override public void initialize(final InputSplit genericSplit, final TaskAttemptContext context) throws IOException { final FileSplit split = (FileSplit) genericSplit; final Configuration configuration = context.getConfiguration(); if (configuration.get(Constants.GREMLIN_HADOOP_GRAPH_FILTER, null) != null) this.graphFilter = VertexProgramHelper.deserialize(ConfUtil.makeApacheConfiguration(configuration), Constants.GREMLIN_HADOOP_GRAPH_FILTER); this.gryoReader = GryoReader.build().mapper( GryoMapper.build().addRegistries(IoRegistryHelper.createRegistries(ConfUtil.makeApacheConfiguration(configuration))).create()).create(); long start = split.getStart(); final Path file = split.getPath(); if (null != new CompressionCodecFactory(configuration).getCodec(file)) { throw new IllegalStateException("Compression is not supported for the (binary) Gryo format"); } // open the file and seek to the start of the split this.inputStream = file.getFileSystem(configuration).open(split.getPath()); this.splitLength = split.getLength(); if (this.splitLength > 0) this.splitLength -= (seekToHeader(this.inputStream, start) - start); }
/** * Create the {@code GryoPool} from this builder. * * @return the new pool */ public GryoPool create() { final GryoMapper.Builder mapper = GryoMapper.build().version(version); final GryoPool gryoPool = new GryoPool(); if (null != this.ioRegistries) this.ioRegistries.forEach(mapper::addRegistry); if (null != this.gryoMapperConsumer) this.gryoMapperConsumer.accept(mapper); gryoPool.createPool(this.poolSize, this.type, mapper.create()); return gryoPool; } }
public static Builder build() { return new Builder(); }
public GryoRecordWriter(final DataOutputStream outputStream, final Configuration configuration) { this.outputStream = outputStream; this.hasEdges = configuration.getBoolean(Constants.GREMLIN_HADOOP_GRAPH_WRITER_HAS_EDGES, true); this.gryoWriter = GryoWriter.build().mapper( GryoMapper.build().addRegistries(IoRegistryHelper.createRegistries(ConfUtil.makeApacheConfiguration(configuration))).create()).create(); }
/** * Creates an instance with a standard {@link GryoMapper} instance. Note that this instance * will be overridden by {@link #configure} is called. */ public GryoLiteMessageSerializerV1d0() { super(overrideWithLite(GryoMapper.build()).create()); }
private static GryoMapper.Builder overrideWithLite(final GryoMapper.Builder builder) { // override the core graph Elements so as to serialize with "reference" as opposed to "detached" builder.addCustom(Edge.class, new EdgeLiteSerializer()); builder.addCustom(Vertex.class, new VertexLiteSerializer()); builder.addCustom(VertexProperty.class, new VertexPropertyLiteSerializer()); builder.addCustom(Property.class, new PropertyLiteSerializer()); builder.addCustom(Path.class, new PathLiteSerializer()); return builder; }
/** * {@inheritDoc} */ @Override public GryoWriter.Builder writer() { return GryoWriter.build().mapper(mapper().create()); }
public synchronized static void initialize(final Configuration configuration) { if (!INITIALIZED) { INITIALIZED = true; GRYO_POOL = GryoPool.build(). poolSize(configuration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, 256)). version(GryoVersion.valueOf(configuration.getString(GryoPool.CONFIG_IO_GRYO_VERSION, GryoPool.CONFIG_IO_GRYO_POOL_VERSION_DEFAULT.name()))). ioRegistries(configuration.getList(IoRegistry.IO_REGISTRY, Collections.emptyList())). initializeMapper(m -> m.registrationRequired(false)). create(); } }
/** * Register custom classes to serializes with gryo using default serialization. Note that calling this method * for a class that is already registered will override that registration. */ public Builder addCustom(final Class... custom) { if (custom != null && custom.length > 0) { for (Class c : custom) { addOrOverrideRegistration(c, id -> GryoTypeReg.of(c, id)); } } return this; }