@Override protected TileSerializer<T> create () throws ConfigurationException { return new PrimitiveAvroSerializer<>(_type, TileSerializerFactory.getCodecFactory(this)); } }
TileSerializer<Double> serializer = new PrimitiveAvroSerializer<Double>(Double.class, CodecFactory.bzip2Codec()); TileExporter extractor = new TileExporter(zookeeperQuorum, zookeeperPort, hbaseMaster, rootPath, extension); extractor.copyPyramid(pyramidId, minLevel, maxLevel, serializer, blockSize);
@Test public void testAvroTileDeSerialization () throws Exception { deserialize(AVRO, SCALAR_DATA, _scalarData, new PrimitiveAvroSerializer<>(Double.class, CodecFactory.bzip2Codec())); }
@Test public void testAvroTileSerialization () throws Exception { serialize(AVRO, SCALAR_DATA, _scalarData, new PrimitiveAvroSerializer<>(Double.class, CodecFactory.bzip2Codec())); }
@Test public void testAvroMetaDataSerialization () throws Exception { TileSerializer<Double> serializer = new PrimitiveAvroSerializer<>(Double.class, CodecFactory.nullCodec()); ByteArrayOutputStream output = new ByteArrayOutputStream(); serializer.serialize(_tile, output); output.flush(); output.close(); ByteArrayInputStream input = new ByteArrayInputStream(output.toByteArray()); TileData<Double> received = serializer.deserialize(_index, input); Assert.assertEquals(2, received.getMetaDataProperties().size()); Assert.assertTrue(received.getMetaDataProperties().contains("a")); Assert.assertTrue(received.getMetaDataProperties().contains("b")); Assert.assertEquals("abc", received.getMetaData("a")); Assert.assertEquals("bcd", received.getMetaData("b")); }
TileSerializer<Double> serializer = new PrimitiveAvroSerializer<Double>(Double.class, CodecFactory.nullCodec());
@SafeVarargs final <T> void testRoundTrip(Class<? extends T> type, T... data) throws Exception { TileSerializer<T> serializer = new PrimitiveAvroSerializer<T>(type, CodecFactory.nullCodec()); // Create our tile int size = (int) Math.ceil(Math.sqrt(data.length)); TileData<T> input = new DenseTileData<>(new TileIndex(0, 0, 0, size, size)); for (int y=0; y<size; ++y) { for (int x=0; x<size; ++x) { int i = (x+size*y) % data.length; input.setBin(x, y, data[i]); } } // Send it round-trip through serialization ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(input, baos); baos.flush(); baos.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); TileData<T> output = serializer.deserialize(new TileIndex(1, 1, 1, size, size), bais); // Test to make sure output matches input. Assert.assertEquals(input.getDefinition(), output.getDefinition()); for (int y=0; y<size; ++y) { for (int x=0; x<size; ++x) { Assert.assertEquals(input.getBin(x, y), output.getBin(x, y)); } } }
TileSerializer<Double> dSerializer = new PrimitiveAvroSerializer<>(Double.class, CodecFactory.bzip2Codec()); String singleSliceTable = "twitter-ebola-p1-heatmap"; long startSingle = System.currentTimeMillis();
@Test public void testPrimitiveSerializer () throws Exception { TileSerializer<Double> serialD = new PrimitiveAvroSerializer<>(Double.class, CodecFactory.nullCodec());
@SafeVarargs final <T> void testRoundTrip(Class<? extends T> type, T defaultBin, T... data) throws Exception { TileSerializer<T> serializer = new PrimitiveAvroSerializer<T>(type, CodecFactory.nullCodec()); // Create our tile int size = data.length; TileData<T> input = new SparseTileData<>(new TileIndex(0, 0, 0, size, size), defaultBin); for (int i=0; i<size; ++i) { input.setBin(i, i, data[i]); } // Send it round-trip through serialization ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(input, baos); baos.flush(); baos.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); TileData<T> output = serializer.deserialize(new TileIndex(1, 1, 1, size, size), bais); // Test to make sure output matches input. Assert.assertEquals(input.getDefinition(), output.getDefinition()); for (int y=0; y<size; ++y) { for (int x=0; x<size; ++x) { Assert.assertEquals(input.getBin(x, y), output.getBin(x, y)); if (x == y) Assert.assertEquals(output.getBin(x, y), data[x]); else Assert.assertEquals(output.getBin(x, y), defaultBin); } } }
@Test public void writeReadAvroRoundTripTest () { PyramidIO io = new TestPyramidIO(); TileSerializer<Integer> serializer = new PrimitiveAvroSerializer<>(Integer.class, CodecFactory.nullCodec());
@Test public void testDoubleTileSerialization() throws IOException { TileIndex index = new TileIndex(2, 0, 1, 10, 20); TileData<Double> tile = new DenseTileData<Double>(index); for (int x=0; x<10; ++x) { for (int y=0; y<20; ++y) { tile.setBin(x, y, ((x+10*y)%7)/2.0); } } PyramidIO io = new TestPyramidIO(); TileSerializer<Double> serializer = new PrimitiveAvroSerializer<>(Double.class, CodecFactory.nullCodec()); io.writeTiles(".", serializer, Collections.singleton(tile)); List<TileData<Double>> tilesOut = io.readTiles(".", serializer, Collections.singleton(index)); Assert.assertEquals(1, tilesOut.size()); TileData<Double> firstOut = tilesOut.get(0); Assert.assertEquals(tile.getDefinition(), firstOut.getDefinition()); for (int x = 0; x < tile.getDefinition().getXBins(); ++x) { for (int y = 0; y < tile.getDefinition().getYBins(); ++y) { Assert.assertEquals(tile.getBin(x, y), firstOut.getBin(x, y), 1E-12); } } }
@Test public void writeReadAvroRoundTripTest () { FileBasedPyramidIO io = new FileBasedPyramidIO(new FileSystemPyramidSource(SOURCE_DIR, SOURCE_EXT)); TileSerializer<Integer> serializer = new PrimitiveAvroSerializer<>(Integer.class, CodecFactory.nullCodec()); ArrayList<TileData<Integer>> writeTiles = new ArrayList<>(); TileIndex index = new TileIndex( LEVEL, X_INDEX, Y_INDEX ); TileData<Integer> tile = new DenseTileData<>(index); for (int x=0; x<256; ++x) { for (int y=0; y<256; ++y) { tile.setBin(x, y, x+256*y); } } writeTiles.add(tile); writeAvroTiles(io, serializer, SOURCE_LAYER, writeTiles); List<TileData<Integer>> readTiles = readAvroTiles(io, serializer, SOURCE_LAYER); for (int i=0; i<writeTiles.size(); i++){ TileData<Integer> writeTile = writeTiles.get(i); TileIndex writeTileDef = writeTile.getDefinition(); TileData<Integer> readTile = readTiles.get(i); TileIndex readTileDef = readTile.getDefinition(); Assert.assertEquals(writeTileDef, readTileDef); for (int x = 0; x < writeTile.getDefinition().getXBins(); ++x) { for (int y = 0; y < writeTile.getDefinition().getYBins(); ++y) { Assert.assertEquals(writeTile.getBin(x, y), readTile.getBin(x, y)); } } } }
@Test public void writeReadAvroRoundTripTest () { FileBasedPyramidIO io = new FileBasedPyramidIO(new FileSystemPyramidSource(SOURCE_DIR, SOURCE_EXT)); TileSerializer<Integer> serializer = new PrimitiveAvroSerializer<>(Integer.class, CodecFactory.nullCodec()); ArrayList<TileData<Integer>> writeTiles = new ArrayList<>(); TileIndex index = new TileIndex( LEVEL, X_INDEX, Y_INDEX ); TileData<Integer> tile = new DenseTileData<>(index); for (int x=0; x<256; ++x) { for (int y=0; y<256; ++y) { tile.setBin(x, y, x+256*y); } } writeTiles.add(tile); writeAvroTiles(io, serializer, SOURCE_LAYER, writeTiles); List<TileData<Integer>> readTiles = readAvroTiles(io, serializer, SOURCE_LAYER); for (int i=0; i<writeTiles.size(); i++){ TileData<Integer> writeTile = writeTiles.get(i); TileIndex writeTileDef = writeTile.getDefinition(); TileData<Integer> readTile = readTiles.get(i); TileIndex readTileDef = readTile.getDefinition(); Assert.assertEquals(writeTileDef, readTileDef); for (int x = 0; x < writeTile.getDefinition().getXBins(); ++x) { for (int y = 0; y < writeTile.getDefinition().getYBins(); ++y) { Assert.assertEquals(writeTile.getBin(x, y), readTile.getBin(x, y)); } } } }
case Avro: if (null == _serializer || !(_serializer instanceof PrimitiveAvroSerializer)) { _serializer = new PrimitiveAvroSerializer<Double>(Double.class, CodecFactory.bzip2Codec()); _bucketSerializer = null; changed = true;
@Test public void testSerializerTypeing () { Assert.assertEquals(new TypeDescriptor(Double.class), new PrimitiveAvroSerializer<>(Double.class, CodecFactory.nullCodec()).getBinTypeDescription()); Assert.assertEquals(new TypeDescriptor(List.class, new TypeDescriptor(Double.class)), new PrimitiveArrayAvroSerializer<>(Double.class, CodecFactory.nullCodec()).getBinTypeDescription()); Assert.assertEquals(new TypeDescriptor(List.class, new TypeDescriptor(Pair.class, new TypeDescriptor(String.class), new TypeDescriptor(Integer.class))), new PairArrayAvroSerializer<>(String.class, Integer.class, CodecFactory.nullCodec()).getBinTypeDescription()); } }