@Override public TileIndex getDefinition() { return _tileData.getDefinition(); }
@Override public String getMetaData (String property) { return _data.getMetaData(property); }
private int getBucketCount(TileData<List<?>> data) throws JSONException { int size = data.getBin(0,0).size(); if (size == 0) { // Try to get from metadata String maxArray = data.getMetaData("maximum array"); if (maxArray != null) { size = new JSONArray(maxArray).length(); } } return size; }
protected Map<String, String> getTileMetaData (TileData<T> tile) { Collection<String> keys = tile.getMetaDataProperties(); if (null == keys || keys.isEmpty()) return null; Map<String, String> metaData = new HashMap<String, String>(); for (String key: keys) { String value = tile.getMetaData(key); if (null != value) metaData.put(key, value); } return metaData; }
@SafeVarargs final <T> void testRoundTrip(Class<? extends T> type, T defaultBin, T... data) throws Exception { TileSerializer<T> serializer = new PrimitiveAvroSerializer<T>(type, CodecFactory.nullCodec()); // Create our tile int size = data.length; TileData<T> input = new SparseTileData<>(new TileIndex(0, 0, 0, size, size), defaultBin); for (int i=0; i<size; ++i) { input.setBin(i, i, data[i]); } // Send it round-trip through serialization ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(input, baos); baos.flush(); baos.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); TileData<T> output = serializer.deserialize(new TileIndex(1, 1, 1, size, size), bais); // Test to make sure output matches input. Assert.assertEquals(input.getDefinition(), output.getDefinition()); for (int y=0; y<size; ++y) { for (int x=0; x<size; ++x) { Assert.assertEquals(input.getBin(x, y), output.getBin(x, y)); if (x == y) Assert.assertEquals(output.getBin(x, y), data[x]); else Assert.assertEquals(output.getBin(x, y), defaultBin); } } }
@Override public void serialize (TileData<T> tile, OutputStream stream) throws IOException { TileIndex tileIndex = tile.getDefinition(); int xBins = tile.getDefinition().getXBins(); int yBins = tile.getDefinition().getYBins(); for (int y = 0; y < yBins; ++y) { for (int x = 0; x < xBins; ++x) { T value = tile.getBin(x, y); if (value == null) { bins.put( new JSONObject() ); Collection<String> keys = tile.getMetaDataProperties(); if (null != keys) { for (String key: keys) { String value = tile.getMetaData(key); if (null != value) metaData.put(key, value);
/** * Get all the data for a given tile, in a form that can be used to initialize a dense tile. */ static public <T> List<T> getData (TileData<T> tile) { if (tile instanceof DenseTileData) { return ((DenseTileData<T>) tile).getData(); } else { List<T> result = new ArrayList<>(); TileIndex idx = tile.getDefinition(); for (int y = 0; y < idx.getYBins(); ++y) { for (int x = 0; x < idx.getXBins(); ++x) { result.add(tile.getBin(x, y)); } } return result; } }
@Test public void testMetaDataSerialization () throws Exception { TileIndex index = new TileIndex(0, 0, 0, 2, 2); TileData<Double> tile = new DenseTileData<>(index); tile.setBin(0, 0, 1.0); tile.setBin(0, 1, 2.0); tile.setBin(1, 0, 3.0); tile.setBin(1, 1, 4.0); tile.setMetaData("a", "abc"); tile.setMetaData("b", "bcd"); TileSerializer<Double> serializer = new KryoSerializer<Double>(new TypeDescriptor(Double.class)); ByteArrayOutputStream output = new ByteArrayOutputStream(); serializer.serialize(tile, output); output.flush(); output.close(); ByteArrayInputStream input = new ByteArrayInputStream(output.toByteArray()); TileData<Double> received = serializer.deserialize(index, input); Assert.assertEquals(2, received.getMetaDataProperties().size()); Assert.assertTrue(received.getMetaDataProperties().contains("a")); Assert.assertTrue(received.getMetaDataProperties().contains("b")); Assert.assertEquals("abc", received.getMetaData("a")); Assert.assertEquals("bcd", received.getMetaData("b")); }
@Override public void setMetaData (String property, Object value) { _data.setMetaData(property, value); } }
@Override public List<T> getBin (int x, int y) { return Arrays.asList(_data.getBin(x, y)); }
@Override public TileData<List<T>> transform (TileData<List<T>> inputData) throws Exception { TileData<List<T>> resultTile = inputData; // add in metadata to the tile JSONObject metadata = new JSONObject(inputData.getMetaData("meta")); JSONObject filteredMetadata = null; if ( metadata.length() > 0 ) { filteredMetadata = filterKeywordMetadata(metadata); if (null != filteredMetadata) { resultTile.setMetaData("meta", filteredMetadata); } } return resultTile; }
private void serializeDense (TileData<T> tile, OutputStream stream) throws IOException { Schema recordSchema = getRecordSchema(); Schema tileSchema = getTileSchema(StorageType.Dense); TileIndex idx = tile.getDefinition(); List<GenericRecord> bins = new ArrayList<GenericRecord>(); List<T> denseData = DenseTileData.getData(tile); for (T value: denseData) { GenericRecord bin = new GenericData.Record(recordSchema); setValue(bin, value); bins.add(bin); } GenericRecord tileRecord = new GenericData.Record(tileSchema); tileRecord.put("level", idx.getLevel()); tileRecord.put("xIndex", idx.getX()); tileRecord.put("yIndex", idx.getY()); tileRecord.put("xBinCount", idx.getXBins()); tileRecord.put("yBinCount", idx.getYBins()); tileRecord.put("values", bins); tileRecord.put("meta", getTileMetaData(tile)); T defaultValue = tile.getDefaultValue(); if (null == defaultValue) { tileRecord.put("default", null); } else { GenericRecord defaultValueRecord = new GenericData.Record(recordSchema); setValue(defaultValueRecord, tile.getDefaultValue()); tileRecord.put("default", defaultValueRecord); } writeRecord(tileRecord, tileSchema, stream); }
@Override public List<T> getDefaultValue () { return _data.getDefaultValue(); }
@Override public Collection<String> getMetaDataProperties() { return _tileData.getMetaDataProperties(); }
@Before public void setup () { _index = new TileIndex(0, 0, 0, 2, 2); _tile = new DenseTileData<>(_index); _tile.setBin(0, 0, 1.0); _tile.setBin(0, 1, 2.0); _tile.setBin(1, 0, 3.0); _tile.setBin(1, 1, 4.0); _tile.setMetaData("a", "abc"); _tile.setMetaData("b", "bcd"); }
@Override public void setBin(int x, int y, List<T> value) { if (x < 0 || x >= getDefinition().getXBins()) { throw new IllegalArgumentException("Bin x index is outside of tile's valid bin range"); } if (y < 0 || y >= getDefinition().getYBins()) { throw new IllegalArgumentException("Bin y index is outside of tile's valid bin range"); } _base.setBin( x, y, value ); }
@Test public void testDoubleTileSerialization() throws IOException { TileIndex index = new TileIndex(2, 0, 1, 10, 20); TileData<Double> tile = new DenseTileData<Double>(index); for (int x=0; x<10; ++x) { for (int y=0; y<20; ++y) { tile.setBin(x, y, ((x+10*y)%7)/2.0); } } PyramidIO io = new TestPyramidIO(); TileSerializer<Double> serializer = new PrimitiveAvroSerializer<>(Double.class, CodecFactory.nullCodec()); io.writeTiles(".", serializer, Collections.singleton(tile)); List<TileData<Double>> tilesOut = io.readTiles(".", serializer, Collections.singleton(index)); Assert.assertEquals(1, tilesOut.size()); TileData<Double> firstOut = tilesOut.get(0); Assert.assertEquals(tile.getDefinition(), firstOut.getDefinition()); for (int x = 0; x < tile.getDefinition().getXBins(); ++x) { for (int y = 0; y < tile.getDefinition().getYBins(); ++y) { Assert.assertEquals(tile.getBin(x, y), firstOut.getBin(x, y), 1E-12); } } }
private TileData<Double> addTiles (TileData<Double> a, TileData<Double> b) { int xBins = a.getDefinition().getXBins(); int yBins = a.getDefinition().getYBins(); List<Double> dc = new ArrayList<>(); for (int y = 0; y < yBins; ++y) { for (int x = 0; x < xBins; ++x) { dc.add(a.getBin(x, y) + b.getBin(x, y)); } } return new DenseTileData<Double>(a.getDefinition(), dc); }
@Override public void setMetaData (String property, Object value) { _data.setMetaData(property, value); } }