@Override public TileData<Double> deserialize(TileIndex index, InputStream rawData) throws IOException { ZipInputStream stream = new ZipInputStream(rawData); stream.getNextEntry(); ObjectInputStream ois = new ObjectInputStream(stream); try { double[] data = (double[]) ois.readObject(); List<Double> d = Doubles.asList(data); TileData<Double> tileData = new DenseTileData<Double>(index, d); return tileData; } catch (ClassNotFoundException e) { e.printStackTrace(); } return null; }
static public AnnotationTile getTileFromJSON( JSONObject json ) throws IllegalArgumentException { try { TileIndex index = new TileIndex( json.getInt("level"), json.getInt("x"), json.getInt("y"), AnnotationIndexer.NUM_BINS, AnnotationIndexer.NUM_BINS ); // create tile with empty bins DenseTileData<AnnotationBin> tile = new DenseTileData<>( index ); // for all binkeys Iterator<?> binKeys = json.keys(); while( binKeys.hasNext() ) { String binKey = (String)binKeys.next(); if( json.get(binKey) instanceof JSONObject ){ JSONObject bin = (JSONObject)json.get(binKey); BinIndex binIndex = BinIndex.fromString( binKey ); tile.setBin( binIndex.getX(), binIndex.getY(), getBinFromJSON( bin )); } } return new AnnotationTile( tile.getDefinition(), tile.getData() ); } catch ( Exception e ) { throw new IllegalArgumentException( e ); } }
@Test public void testDenseDefaults () throws Exception { TileSerializer<List<Integer>> serializer = new PrimitiveArrayAvroSerializer<>(Integer.class, CodecFactory.nullCodec()); DenseTileData<List<Integer>> denseBase = new DenseTileData<List<Integer>>(new TileIndex(0, 0, 0, 2, 2), Arrays.asList(1, 2, 3)); { ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(denseBase, baos); baos.flush(); baos.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); TileData<List<Integer>> out = serializer.deserialize(denseBase.getDefinition(), bais); Assert.assertTrue(out instanceof DenseTileData<?>); List<Integer> baseDefault = denseBase.getDefaultValue(); List<Integer> readDefault = ((DenseTileData<List<Integer>>) out).getDefaultValue(); Assert.assertEquals(baseDefault.size(), readDefault.size()); for (int i=0; i<baseDefault.size(); ++i) { Assert.assertEquals(baseDefault.get(i), readDefault.get(i)); } } }
@Override public TileData<List<T>> transform (TileData<List<T>> inputData) throws Exception { //list of indices to keep TileData<List<T>> resultTile; //If there are none to keep, return empty list if (_variables == null) { resultTile = null; } else { TileIndex index = inputData.getDefinition(); List<List<T>> rawData = DenseTileData.getData(inputData); List<List<T>> transformedData = new ArrayList<>(index.getXBins()*index.getYBins()); for (List<T> rawEntry: rawData) { int size = rawEntry.size(); List<T> transformedEntry = new ArrayList<>(_variables.size()); for (int varIndex : _variables) { if (varIndex < size) transformedEntry.add(rawEntry.get(varIndex)); } transformedData.add(transformedEntry); } resultTile = new DenseTileData<>(inputData.getDefinition(), transformedData); } return resultTile; }
public TileData<List<T>> harden () { TileIndex index = getDefinition(); if (isDense()) { DenseTileData<List<T>> hardened = new DenseTileData<>(index); hardened.setDefaultValue(_base.getDefaultValue()); for (int x=0; x<index.getXBins(); ++x) { for (int y=0; y<index.getYBins(); ++y) { hardened.setBin(x, y, getBin(x, y)); } } return hardened; } else { SparseTileData<List<T>> hardened = new SparseTileData<>(index, _base.getDefaultValue()); for (int x=0; x<index.getXBins(); ++x) { for (int y=0; y<index.getYBins(); ++y) { List<T> value = getBin(x, y); if (null != value) { hardened.setBin(x, y, getBin(x, y)); } } } return hardened; } }
@Test public void testReadWordScoreTile () throws IOException, JSONException { // Create a tile to test TileSerializer<List<Pair<String, Double>>> serializer = new PairArrayAvroSerializer<>(String.class, Double.class, CodecFactory.nullCodec()); TileIndex index = new TileIndex(0, 0, 0, 1, 1); DenseTileData<List<Pair<String, Double>>> tile = new DenseTileData<>(index); List<Pair<String, Double>> bin = new ArrayList<>(); bin.add(new Pair<String, Double>("abc", 1.0)); bin.add(new Pair<String, Double>("def", 1.5)); bin.add(new Pair<String, Double>("ghi", 2.0)); bin.add(new Pair<String, Double>("jkl", 2.25)); tile.setBin(0, 0, bin); ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(tile, baos); baos.flush(); baos.close(); byte[] serializedTileData = baos.toByteArray(); // Now try to convert that to JSON. JSONObject result = AvroJSONConverter.convert(new ByteArrayInputStream(serializedTileData)); System.out.println(result.toString()); } }
private double[] getTileData (TileData<Double> tile) { List<Double> data = DenseTileData.getData(tile); double[] result = new double[data.size()]; for (int i=0; i<data.size(); ++i) { result[i] = ((Number) data.get(i)).doubleValue(); } return result; } }
@Override public String toString () { return "<dense-tile index=\""+getDefinition()+"\", default=\""+_default+"\"/>"; } }
public static List< AnnotationBin > convertFromRaw( TileData<Map<String, List<Pair<String, Long>>>> rawTile ) { List<Map<String, List<Pair<String, Long>>>> rawData = DenseTileData.getData(rawTile); List< AnnotationBin > bins = new ArrayList<>(); for (Map<String, List<Pair<String, Long>>> rawBin: rawData) { if ( rawBin != null ) { bins.add( new AnnotationBin( rawBin ) ); } else { bins.add( null ); } } return bins; }
@Override public <T> List<TileData<T>> readTiles (String pyramidId, TileSerializer<T> serializer, Iterable<TileIndex> tiles) throws IOException { List<TileData<T>> results = new ArrayList<>(); for (TileIndex rawIndex: tiles) { TileIndex index = new TileIndex(rawIndex.getLevel(), rawIndex.getX(), rawIndex.getY(), 1, 1); TileData<T> tile = new DenseTileData<>(index); results.add(tile); } return results; }
/** * Get all the data for a given tile, in a form that can be used to initialize a dense tile. */ static public <T> List<T> getData (TileData<T> tile) { if (tile instanceof DenseTileData) { return ((DenseTileData<T>) tile).getData(); } else { List<T> result = new ArrayList<>(); TileIndex idx = tile.getDefinition(); for (int y = 0; y < idx.getYBins(); ++y) { for (int x = 0; x < idx.getXBins(); ++x) { result.add(tile.getBin(x, y)); } } return result; } }
@Override public <T> InputStream getTileStream (String pyramidId, TileSerializer<T> serializer, TileIndex rawIndex) throws IOException { TileIndex index = new TileIndex(rawIndex.getLevel(), rawIndex.getX(), rawIndex.getY(), 1, 1); ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.serialize(new DenseTileData<T>(index), baos); return new ByteArrayInputStream(baos.toByteArray()); }
private void serializeDense (TileData<T> tile, OutputStream stream) throws IOException { Schema recordSchema = getRecordSchema(); Schema tileSchema = getTileSchema(StorageType.Dense); TileIndex idx = tile.getDefinition(); List<GenericRecord> bins = new ArrayList<GenericRecord>(); List<T> denseData = DenseTileData.getData(tile); for (T value: denseData) { GenericRecord bin = new GenericData.Record(recordSchema); setValue(bin, value); bins.add(bin); } GenericRecord tileRecord = new GenericData.Record(tileSchema); tileRecord.put("level", idx.getLevel()); tileRecord.put("xIndex", idx.getX()); tileRecord.put("yIndex", idx.getY()); tileRecord.put("xBinCount", idx.getXBins()); tileRecord.put("yBinCount", idx.getYBins()); tileRecord.put("values", bins); tileRecord.put("meta", getTileMetaData(tile)); T defaultValue = tile.getDefaultValue(); if (null == defaultValue) { tileRecord.put("default", null); } else { GenericRecord defaultValueRecord = new GenericData.Record(recordSchema); setValue(defaultValueRecord, tile.getDefaultValue()); tileRecord.put("default", defaultValueRecord); } writeRecord(tileRecord, tileSchema, stream); }
TileData<T> tile = new DenseTileData<T>( tileIndex, values);
public TileData<Map<String, List<Pair<String,Long>>>> getRawData() { List< Map<String, List<Pair<String,Long>>> > rawBins = AnnotationBin.convertToRaw( getData() ); return new DenseTileData<>( getDefinition(), rawBins ); }
newTile = new DenseTileData<T>(newTileIndex, defaultValue, data); break;
@Before public void setup () { // Create some data Random random = new Random(15485863); _scalarData = new DenseTileData<>(new TileIndex(0, 0, 0, 256, 256)); for (int x=0; x < BIN_SIZE; ++x) { for (int y=0; y < BIN_SIZE; ++y) { _scalarData.setBin(x, y, random.nextDouble()); } } // Random vector data for more complex types. _vectorData = new DenseTileData<>(new TileIndex(0, 0, 0, 256, 256)); for (int x=0; x < BIN_SIZE; ++x) { for (int y=0; y < BIN_SIZE; ++y) { List<Double> binData = new ArrayList<Double>(1); for (int i = 0; i < VECTOR_SIZE; i++) { binData.add(random.nextBoolean() ? random.nextDouble() : 0.0); } _vectorData.setBin(x, y, binData); } } }
@Test public void testUnmodifiableBinReplacement () { List<Integer> binVal = new ArrayList<>(); binVal.add(3); binVal.add(4); binVal.add(5); TileData<List<Integer>> base = new DenseTileData<List<Integer>>(new TileIndex(0, 0, 0, 1, 1)); base.setBin(0, 0, Collections.unmodifiableList(binVal)); TileData<List<Integer>> slice = new DenseTileMultiSliceView<Integer>(base, Arrays.asList(2, 0)); slice.setBin(0, 0, Arrays.asList(6, 2)); Assert.assertEquals(2, base.getBin(0, 0).get(0).intValue()); Assert.assertEquals(4, base.getBin(0, 0).get(1).intValue()); Assert.assertEquals(6, base.getBin(0, 0).get(2).intValue()); Assert.assertEquals(3, binVal.get(0).intValue()); Assert.assertEquals(4, binVal.get(1).intValue()); Assert.assertEquals(5, binVal.get(2).intValue()); }
@Test public void testModifiableBinReplacement () { List<Integer> binVal = new ArrayList<>(); binVal.add(3); binVal.add(4); binVal.add(5); TileData<List<Integer>> base = new DenseTileData<List<Integer>>(new TileIndex(0, 0, 0, 1, 1)); base.setBin(0, 0, binVal); TileData<List<Integer>> slice = new DenseTileMultiSliceView<Integer>(base, Arrays.asList(2, 0)); slice.setBin(0, 0, Arrays.asList(6, 2)); Assert.assertEquals(2, base.getBin(0, 0).get(0).intValue()); Assert.assertEquals(4, base.getBin(0, 0).get(1).intValue()); Assert.assertEquals(6, base.getBin(0, 0).get(2).intValue()); Assert.assertEquals(2, binVal.get(0).intValue()); Assert.assertEquals(4, binVal.get(1).intValue()); Assert.assertEquals(6, binVal.get(2).intValue()); }
private TileData<Double> createRandomTile () { TileIndex index = new TileIndex(0, 0, 0); TileData<Double> datum = new DenseTileData<>(index); for (int x=0; x<index.getXBins(); ++x) { for (int y=0; y<index.getYBins(); ++y) { datum.setBin(x, y, Math.random()); } } return datum; } private TileData<Double> addTiles (TileData<Double> a, TileData<Double> b) {