@Override protected TileSerializer<T> create () throws ConfigurationException { // Check class registrations List<String> classNames = getPropertyValue(NEEDED_CLASSES); Codec codec = getPropertyValue(CODEC); int nc = classNames.size(); Class<?>[] classes = new Class<?>[nc]; for (int n=0; n<nc; ++n) { try { classes[n] = Class.forName(classNames.get(n)); } catch (ClassNotFoundException e) { LOGGER.warn("Class {} not found", classNames.get(n)); } } return new KryoSerializer<>(_type, codec, classes); } }
@Test public void testGZip () throws Exception { // Create our serializer KryoSerializer<List<Integer>> serializer = new KryoSerializer<>(new TypeDescriptor(List.class, new TypeDescriptor(Integer.class)), KryoSerializer.Codec.GZIP); // Create a large tile int xN = 256; int yN = 256; int zN = 100; TileIndex index = new TileIndex(0, 0, 0, xN, yN); TileData<List<Integer>> input = new DenseTileData<List<Integer>>(index); for (int x=0; x<xN; ++x) { for (int y=0; y<yN; ++y) { List<Integer> bin = new ArrayList<>(zN); for (int z=0; z<zN; ++z) bin.add((int) Math.floor(Math.random()*1024)); input.setBin(x, y, bin); } } // Write it out ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(input, baos); baos.flush(); baos.close(); // Read it back in ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); TileData<List<Integer>> output = serializer.deserialize(index, bais); assertListTilesEqual(input, output); }
@Override public void serialize(TileData<T> data, OutputStream stream) throws IOException { OutputStream compressionStream; switch (_codec) { case BZIP: compressionStream = new BZip2CompressorOutputStream(stream); break; case GZIP: compressionStream = new GzipCompressorOutputStream(stream); break; case DEFLATE: default: compressionStream = new DeflaterOutputStream(stream); break; } Output output = new Output(compressionStream); try { kryo().writeClassAndObject(output, data); output.flush(); compressionStream.flush(); } finally { output.close(); compressionStream.close(); } }
@Test public void testSerializerSerializability () throws IOException, ClassNotFoundException, ConfigurationException { KryoSerializer<CustomTestData> input = new KryoSerializer<>(new TypeDescriptor(CustomTestData.class), CustomTestData.class); Assert.assertEquals(new TypeDescriptor(CustomTestData.class), genericOutput.getBinTypeDescription()); TileSerializer<CustomTestData> output = SerializationTypeChecker.checkBinClass(genericOutput, CustomTestData.class, input.serialize(inputData, baos); baos.flush(); baos.close(); baos.close(); bais = new ByteArrayInputStream(baos.toByteArray()); TileData<CustomTestData> outputData2 = input.deserialize(new TileIndex(0, 0, 0, 1, 1), bais);
@Test public void testCompression () throws IOException { TileData<Double> data = new DenseTileData<Double>(new TileIndex(0, 0, 0), 1.1); TileSerializer<Double> serializer = new KryoSerializer<Double>(new TypeDescriptor(Double.class)); ByteArrayOutputStream output = new ByteArrayOutputStream(); serializer.serialize(data, output); output.flush(); output.close(); byte[] buffer = output.toByteArray(); Assert.assertTrue(buffer.length < 256*256); }
new KryoSerializer<>(new TypeDescriptor(List.class, new TypeDescriptor(Integer.class)), KryoSerializer.Codec.DEFLATE); serializer.serialize(input, baos); baos.flush(); baos.close(); TileData<List<Integer>> output = serializer.deserialize(index, bais);
@SuppressWarnings({ "rawtypes", "unchecked" }) @Override public TileData<T> deserialize(TileIndex index, InputStream stream) throws IOException { InputStream compressionStream; switch (_codec) { case BZIP: compressionStream = new BZipInputStreamWrapper(new BZip2CompressorInputStream(stream)); break; case GZIP: compressionStream = new GzipCompressorInputStream(stream); break; case DEFLATE: default: compressionStream = new InflaterInputStream(stream); break; } Input input = new Input(compressionStream); try { Object data = kryo().readClassAndObject(input); if (data instanceof TileData) return (TileData) data; else return null; } finally { compressionStream.close(); input.close(); } }
@Test public void testKryoCustomClassSpecification () throws Exception { TypeDescriptor testType = new TypeDescriptor(TestClass.class); KryoSerializerFactory<TestClass> factory = new KryoSerializerFactory<TestClass>(null, Arrays.asList("factory"), testType); factory.readConfiguration(new JSONObject( "{\"factory\": {\n"+ " \"codec\": \"DEFLATE\",\n"+ " \"classes\": [\""+TestClass.class.getName()+"\"]\n"+ "}}")); TileSerializer<TestClass> product = SerializationTypeChecker.checkBinClass(factory.produce(TileSerializer.class), TestClass.class, testType); TileSerializer<TestClass> expected = new KryoSerializer<TestClass>(testType, Codec.DEFLATE, TestClass.class); assertSerializersEqual(expected, product, new TestClassSource()); }
@Test public void testBZip () throws Exception { // Create our serializer KryoSerializer<List<Integer>> serializer = new KryoSerializer<>(new TypeDescriptor(List.class, new TypeDescriptor(Integer.class)), KryoSerializer.Codec.BZIP); // Create a large tile int xN = 256; int yN = 256; int zN = 100; TileIndex index = new TileIndex(0, 0, 0, xN, yN); TileData<List<Integer>> input = new DenseTileData<List<Integer>>(index); for (int x=0; x<xN; ++x) { for (int y=0; y<yN; ++y) { List<Integer> bin = new ArrayList<>(zN); for (int z=0; z<zN; ++z) bin.add((int) Math.floor(Math.random()*1024)); input.setBin(x, y, bin); } } // Write it out ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(input, baos); baos.flush(); baos.close(); // Read it back in ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); TileData<List<Integer>> output = serializer.deserialize(index, bais); assertListTilesEqual(input, output); }
@Test public void testKryoTileDeSerialization () throws Exception { deserialize(KRYO, SCALAR_DATA, _scalarData, new KryoSerializer<Double>(new TypeDescriptor(Double.class))); }
@Test public void testKryoTileSerialization () throws Exception { serialize(KRYO, SCALAR_DATA, _scalarData, new KryoSerializer<Double>(new TypeDescriptor(Double.class))); }
@Test public void testKryoDefaultCodecSpecification () throws Exception { TypeDescriptor integerType = new TypeDescriptor(Integer.class); KryoSerializerFactory<Integer> factory = new KryoSerializerFactory<Integer>(null, Arrays.asList("factory"), integerType); factory.readConfiguration(new JSONObject("{}")); TileSerializer<Integer> product = SerializationTypeChecker.checkBinClass(factory.produce(TileSerializer.class), Integer.class, integerType); TileSerializer<Integer> expected = new KryoSerializer<Integer>(integerType, Codec.GZIP); assertSerializersEqual(expected, product, new IntegerSource()); }
@Test public void testKryoGZipSpecification () throws Exception { TypeDescriptor integerType = new TypeDescriptor(Integer.class); KryoSerializerFactory<Integer> factory = new KryoSerializerFactory<Integer>(null, Arrays.asList("factory"), integerType); factory.readConfiguration(new JSONObject("{\"factory\": { \"codec\": \"GZIP\"}}")); TileSerializer<Integer> product = SerializationTypeChecker.checkBinClass(factory.produce(TileSerializer.class), Integer.class, integerType); TileSerializer<Integer> expected = new KryoSerializer<Integer>(integerType, Codec.GZIP); assertSerializersEqual(expected, product, new IntegerSource()); }
@Test public void testKryoBZip2Specification () throws Exception { TypeDescriptor integerType = new TypeDescriptor(Integer.class); KryoSerializerFactory<Integer> factory = new KryoSerializerFactory<Integer>(null, Arrays.asList("factory"), integerType); factory.readConfiguration(new JSONObject("{\"factory\": { \"codec\": \"BZIP\"}}")); TileSerializer<Integer> product = SerializationTypeChecker.checkBinClass(factory.produce(TileSerializer.class), Integer.class, integerType); TileSerializer<Integer> expected = new KryoSerializer<Integer>(integerType, Codec.BZIP); assertSerializersEqual(expected, product, new IntegerSource()); }
@Test public void testKryoDeflateSpecification () throws Exception { TypeDescriptor integerType = new TypeDescriptor(Integer.class); KryoSerializerFactory<Integer> factory = new KryoSerializerFactory<Integer>(null, Arrays.asList("factory"), integerType); factory.readConfiguration(new JSONObject("{\"factory\": { \"codec\": \"DEFLATE\"}}")); TileSerializer<Integer> product = SerializationTypeChecker.checkBinClass(factory.produce(TileSerializer.class), Integer.class, integerType); TileSerializer<Integer> expected = new KryoSerializer<Integer>(integerType, Codec.DEFLATE); assertSerializersEqual(expected, product, new IntegerSource()); }
String table = "nycTaxiHeatmap_sw2015_weekly"; HBaseSlicedPyramidIO io = new HBaseSlicedPyramidIO("hadoop-s1", "2181", "hadoop-s1:60000"); TileSerializer<List<Integer>> serializer = new KryoSerializer<>(new TypeDescriptor(List.class, new TypeDescriptor(Integer.class))); TileSerializer<List<Integer>> ks = new KryoSerializer<List<Integer>>(new TypeDescriptor(List.class, new TypeDescriptor(Integer.class))); TileIndex index = new TileIndex(9, 151, 319); List<TileIndex> indices = Arrays.asList(index);
case Kryo: if (null == _serializer || !(_serializer instanceof KryoSerializer)) { _serializer = new KryoSerializer<Double>(new TypeDescriptor(Double.class)); _bucketSerializer = null; changed = true; case KryoBucketted: if (null == _bucketSerializer|| !(_bucketSerializer instanceof KryoSerializer)) { _bucketSerializer = new KryoSerializer<>(new TypeDescriptor(List.class, new TypeDescriptor(Double.class)), KryoSerializer.Codec.GZIP); _serializer = null; changed = true;
@SafeVarargs final <T> void testRoundTripDense(TypeDescriptor type, Class<?>[] classesToRegister, T... data) throws Exception { TileSerializer<T> serializer = new KryoSerializer<T>(type, classesToRegister); // Create our tile int size = (int) Math.ceil(Math.sqrt(data.length)); TileData<T> input = new DenseTileData<T>(new TileIndex(0, 0, 0, size, size)); for (int y=0; y<size; ++y) { for (int x=0; x<size; ++x) { int i = (x+size*y) % data.length; input.setBin(x, y, data[i]); } } // Send it round-trip through serialization ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(input, baos); baos.flush(); baos.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); TileData<T> output = serializer.deserialize(new TileIndex(1, 1, 1, size, size), bais); // Test to make sure output matches input. Assert.assertEquals(input.getDefinition(), output.getDefinition()); for (int y=0; y<size; ++y) { for (int x=0; x<size; ++x) { Assert.assertEquals(input.getBin(x, y), output.getBin(x, y)); } } }
@SafeVarargs final <T> void testRoundTripSparse(TypeDescriptor type, Class<?>[] classesToRegister, T defaultValue, T... data) throws Exception { TileSerializer<T> serializer = new KryoSerializer<T>(type, classesToRegister); // Create our tile int size = (int) Math.ceil(Math.sqrt(data.length*2)); TileData<T> input = new SparseTileData<T>(new TileIndex(0, 0, 0, size, size), defaultValue); int i = 0; for (int y=0; y<size; ++y) { for (int x=0; x<size; ++x) { if (0 == ((x+y)%2) && i < data.length) { input.setBin(x, y, data[i]); ++i; } } } // Send it round-trip through serialization ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(input, baos); baos.flush(); baos.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); TileData<T> output = serializer.deserialize(new TileIndex(1, 1, 1, size, size), bais); // Test to make sure output matches input. Assert.assertEquals(input.getDefinition(), output.getDefinition()); for (int y=0; y<size; ++y) { for (int x=0; x<size; ++x) { Assert.assertEquals(input.getBin(x, y), output.getBin(x, y)); } } }
TileSerializer<List<Integer>> serializer = new KryoSerializer<>(new TypeDescriptor(List.class, new TypeDescriptor(Integer.class)));