@Override protected TileSerializer<List<T>> create () throws ConfigurationException { return new PrimitiveArrayAvroSerializer<>(_entryType, TileSerializerFactory.getCodecFactory(this)); } }
@Test public void testPrimitiveArraySerializer () throws Exception { TileSerializer<List<Float>> serialD = new PrimitiveArrayAvroSerializer<>(Float.class, CodecFactory.nullCodec()); Assert.assertTrue(result instanceof PrimitiveArrayAvroSerializer); PrimitiveArrayAvroSerializer<?> serialR = (PrimitiveArrayAvroSerializer<?>) result; Assert.assertEquals(new TypeDescriptor(List.class, new TypeDescriptor(Float.class)), serialR.getBinTypeDescription()); @SuppressWarnings({ "unchecked", "rawtypes" }) PrimitiveArrayAvroSerializer<Float> serialRD = (PrimitiveArrayAvroSerializer) result; serialRD.serialize(tile, tbaos2); tbaos1.flush(); tbaos1.close(); TileData<List<Float>> out1 = serialRD.deserialize(tile.getDefinition(), tbais1);
@Test public void testSerializerTypeing () { Assert.assertEquals(new TypeDescriptor(Double.class), new PrimitiveAvroSerializer<>(Double.class, CodecFactory.nullCodec()).getBinTypeDescription()); Assert.assertEquals(new TypeDescriptor(List.class, new TypeDescriptor(Double.class)), new PrimitiveArrayAvroSerializer<>(Double.class, CodecFactory.nullCodec()).getBinTypeDescription()); Assert.assertEquals(new TypeDescriptor(List.class, new TypeDescriptor(Pair.class, new TypeDescriptor(String.class), new TypeDescriptor(Integer.class))), new PairArrayAvroSerializer<>(String.class, Integer.class, CodecFactory.nullCodec()).getBinTypeDescription()); } }
@Test public void testAvroVectorTileSerialization () throws Exception { serialize(AVRO, VECTOR_DATA, _vectorData, new PrimitiveArrayAvroSerializer<>(Double.class, CodecFactory.bzip2Codec())); }
@Test public void testAvroVectorTileDeSerialization () throws Exception { deserialize(AVRO, VECTOR_DATA, _vectorData, new PrimitiveArrayAvroSerializer<>(Double.class, CodecFactory.bzip2Codec())); }
@Test public void testDenseDefaults () throws Exception { TileSerializer<List<Integer>> serializer = new PrimitiveArrayAvroSerializer<>(Integer.class, CodecFactory.nullCodec()); DenseTileData<List<Integer>> denseBase = new DenseTileData<List<Integer>>(new TileIndex(0, 0, 0, 2, 2), Arrays.asList(1, 2, 3)); { ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(denseBase, baos); baos.flush(); baos.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); TileData<List<Integer>> out = serializer.deserialize(denseBase.getDefinition(), bais); Assert.assertTrue(out instanceof DenseTileData<?>); List<Integer> baseDefault = denseBase.getDefaultValue(); List<Integer> readDefault = ((DenseTileData<List<Integer>>) out).getDefaultValue(); Assert.assertEquals(baseDefault.size(), readDefault.size()); for (int i=0; i<baseDefault.size(); ++i) { Assert.assertEquals(baseDefault.get(i), readDefault.get(i)); } } }
@Test public void testSparseDefaults () throws Exception { TileSerializer<List<Integer>> serializer = new PrimitiveArrayAvroSerializer<>(Integer.class, CodecFactory.nullCodec()); SparseTileData<List<Integer>> sparseBase = new SparseTileData<List<Integer>>(new TileIndex(0, 0, 0, 2, 2), Arrays.asList(4, 5, 6)); { ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(sparseBase, baos); baos.flush(); baos.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); TileData<List<Integer>> out = serializer.deserialize(sparseBase.getDefinition(), bais); Assert.assertTrue(out instanceof SparseTileData<?>); List<Integer> baseDefault = sparseBase.getDefaultValue(); List<Integer> readDefault = ((SparseTileData<List<Integer>>) out).getDefaultValue(); Assert.assertEquals(baseDefault.size(), readDefault.size()); for (int i=0; i<baseDefault.size(); ++i) { Assert.assertEquals(baseDefault.get(i), readDefault.get(i)); } } } }
inputTile.setBin(2, 2, Arrays.asList( 3, 6, 9)); inputTile.setBin(3, 3, Arrays.asList( 4, 8, 12, 16)); TileSerializer<List<Integer>> serializer = new PrimitiveArrayAvroSerializer<>(Integer.class, CodecFactory.nullCodec());
@Test public void testNullBinsInDenseTile () throws Exception { TileData<List<Integer>> inputTile = new DenseTileData<>(new TileIndex(0, 0, 0, 4, 4)); inputTile.setBin(0, 0, Arrays.asList( 1, 2, 3, 4)); inputTile.setBin(1, 1, Arrays.asList( 2, 4, 6, 8)); inputTile.setBin(2, 2, Arrays.asList( 3, 6, 9, 12)); inputTile.setBin(3, 3, Arrays.asList( 4, 8, 12, 16)); TileSerializer<List<Integer>> serializer = new PrimitiveArrayAvroSerializer<>(Integer.class, CodecFactory.nullCodec()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(inputTile, baos); baos.flush(); baos.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); TileData<List<Integer>> outputTile = serializer.deserialize(new TileIndex(0, 0, 0, 4, 4), bais); for (int x=0; x<4; ++x) { for (int y=0; y<4; ++y) { String binName = "Bin ["+x+", "+y+"]"; List<Integer> input = inputTile.getBin(x, y); List<Integer> output = outputTile.getBin(x, y); if (null == input) { Assert.assertTrue(binName, null == output || output.size() == 0); } else { Assert.assertEquals(binName, input.size(), output.size()); for (int z=0; z<input.size(); ++z) { Assert.assertEquals(input.get(z), output.get(z)); } } } } }
@SafeVarargs final <T> void testRoundTrip(Class<? extends T> type, int arraySize, T... data) throws Exception { TileSerializer<List<T>> serializer = new PrimitiveArrayAvroSerializer<T>(type, CodecFactory.nullCodec());
String table = "hbsioTest"; HBaseSlicedPyramidIO io = new HBaseSlicedPyramidIO("hadoop-s1", "2181", "hadoop-s1:60000"); TileSerializer<List<Integer>> serializer = new PrimitiveArrayAvroSerializer<>(Integer.class, CodecFactory.nullCodec()); try { TileIndex index = new TileIndex(0, 0, 0, 1, 1);
@Test public void testStringArrayTileSerialization() throws IOException { TileIndex index = new TileIndex(2, 0, 1, 10, 20); TileData<List<String>> tile = new DenseTileData<List<String>>(index); for (int x=0; x<10; ++x) { for (int y=0; y<20; ++y) { tile.setBin(x, y, Arrays.asList(String.format("bin [%d, %d]", x, y), "second", "third")); } } PyramidIO io = new TestPyramidIO(); TileSerializer<List<String>> serializer = new PrimitiveArrayAvroSerializer<>(String.class, CodecFactory.nullCodec()); io.writeTiles(".", serializer, Collections.singleton(tile)); List<TileData<List<String>>> tilesOut = io.readTiles(".", serializer, Collections.singleton(index)); Assert.assertEquals(1, tilesOut.size()); TileData<List<String>> firstOut = tilesOut.get(0); Assert.assertEquals(tile.getDefinition(), firstOut.getDefinition()); for (int x = 0; x < tile.getDefinition().getXBins(); ++x) { for (int y = 0; y < tile.getDefinition().getYBins(); ++y) { List<String> inData = tile.getBin(x, y); List<String> outData = firstOut.getBin(x, y); Assert.assertEquals(inData.size(), outData.size()); for (int j=0; j<inData.size(); ++j) { Assert.assertEquals(inData.get(j), outData.get(j)); } } } }
@Test public void testDoubleArrayTileSerialization() throws IOException { TileIndex index = new TileIndex(2, 0, 1, 10, 20); TileData<List<Double>> tile = new DenseTileData<List<Double>>(index); for (int x=0; x<10; ++x) { for (int y=0; y<20; ++y) { tile.setBin(x, y, Arrays.asList(1.0*x, 2.0*y)); } } PyramidIO io = new TestPyramidIO(); TileSerializer<List<Double>> serializer = new PrimitiveArrayAvroSerializer<>(Double.class, CodecFactory.nullCodec()); io.writeTiles(".", serializer, Collections.singleton(tile)); List<TileData<List<Double>>> tilesOut = io.readTiles(".", serializer, Collections.singleton(index)); Assert.assertEquals(1, tilesOut.size()); TileData<List<Double>> firstOut = tilesOut.get(0); Assert.assertEquals(tile.getDefinition(), firstOut.getDefinition()); for (int x = 0; x < tile.getDefinition().getXBins(); ++x) { for (int y = 0; y < tile.getDefinition().getYBins(); ++y) { List<Double> inData = tile.getBin(x, y); List<Double> outData = firstOut.getBin(x, y); Assert.assertEquals(inData.size(), outData.size()); for (int j=0; j<inData.size(); ++j) { Assert.assertEquals(inData.get(j), outData.get(j), 1E-12); } } } }
case AvroBucketted: if (null == _bucketSerializer|| !(_bucketSerializer instanceof PrimitiveArrayAvroSerializer)) { _bucketSerializer = new PrimitiveArrayAvroSerializer<>(Double.class, CodecFactory.bzip2Codec()); _serializer = null; changed = true;