if (recordSchema.getType() == Type.FIXED){ Fixed fixed = (Fixed) datum; return fixed.bytes(); } else if (recordSchema.getType() == Type.BYTES){ return AvroSerdeUtils.getBytesFromByteBuffer((ByteBuffer) datum);
Assert.assertNull(newRec.get("nullU")); Assert.assertArrayEquals(bytedef, ((GenericData.Fixed)newRec.get("fixedF1")).bytes()); Assert.assertArrayEquals(bytedef, ((GenericData.Fixed)newRec.get("fixedF2")).bytes()); Assert.assertArrayEquals(bytedef, ((GenericData.Fixed)newRec.get("fixedF3")).bytes()); Assert.assertArrayEquals(bytedef, ((GenericData.Fixed)newRec.get("fixedU")).bytes()); Assert.assertEquals("S", newRec.get("enumF").toString()); Assert.assertEquals("SS", newRec.get("enumU").toString());
private Object deserializeList(Object datum, Schema recordSchema, ListTypeInfo columnType) throws HaivvreoException { // Need to check the original schema to see if this is actually a Fixed. if(recordSchema.getType().equals(Schema.Type.FIXED)) { // We're faking out Hive to work through a type system impedence mismatch. Pull out the backing array and convert to a list. GenericData.Fixed fixed = (GenericData.Fixed) datum; List<Byte> asList = new ArrayList<Byte>(fixed.bytes().length); for(int j = 0; j < fixed.bytes().length; j++) { asList.add(fixed.bytes()[j]); } return asList; } else if(recordSchema.getType().equals(Schema.Type.BYTES)) { // This is going to be slow... hold on. ByteBuffer bb = (ByteBuffer)datum; List<Byte> asList = new ArrayList<Byte>(bb.capacity()); byte[] array = bb.array(); for(int j = 0; j < array.length; j++) { asList.add(array[j]); } return asList; } else { // An actual list, deser its values List listData = (List) datum; Schema listSchema = recordSchema.getElementType(); List<Object> listContents = new ArrayList<Object>(listData.size()); for(Object obj : listData) { listContents.add(worker(obj, listSchema, columnType.getListElementTypeInfo())); } return listContents; } }
@SuppressWarnings("unchecked") @Override public Object get(final int pos) throws ExecException { Schema s = avroObject.getSchema().getFields().get(pos).schema(); Object o = avroObject.get(pos); switch(s.getType()) { case STRING: // unwrap avro UTF8 encoding return o.toString(); case MAP: return new AvroMapWrapper((Map<CharSequence, Object>) o); case RECORD: return new AvroTupleWrapper<T>((T) o); case ENUM: return o.toString(); case ARRAY: return new AvroBagWrapper<GenericData.Record>( (GenericArray<GenericData.Record>) o); case FIXED: return new DataByteArray(((GenericData.Fixed) o).bytes()); case BYTES: return new DataByteArray(((ByteBuffer) o).array()); case UNION: return getPigObject(o); default: return o; } }
/** * @param o An Avro object to convert to an equivalent type in Pig * @return Equivalent Pig object */ public static Object getPigObject(Object o) { if (o instanceof org.apache.avro.util.Utf8) { return o.toString(); } else if (o instanceof IndexedRecord) { return new AvroTupleWrapper<IndexedRecord>((IndexedRecord) o); } else if (o instanceof GenericArray) { return new AvroBagWrapper<GenericData.Record>( (GenericArray<GenericData.Record>) o); } else if (o instanceof Map) { return new AvroMapWrapper((Map<CharSequence, Object>) o); } else if (o instanceof GenericData.Fixed) { return new DataByteArray(((GenericData.Fixed) o).bytes()); } else if (o instanceof ByteBuffer) { return new DataByteArray(((ByteBuffer) o).array()); } else if (o instanceof GenericEnumSymbol) { return o.toString(); } else { return o; } }
public Fixed(byte[] bytes) { bytes(bytes); }
public Fixed(byte[] bytes) { bytes(bytes); }
public Fixed(Schema schema) { bytes(new byte[schema.getFixedSize()]); } public Fixed(byte[] bytes) { bytes(bytes); }
@Override public GenericData.Fixed read(Decoder decoder, Object reuse) throws IOException { if (reuse instanceof GenericData.Fixed) { GenericData.Fixed reusedFixed = (GenericData.Fixed) reuse; if (reusedFixed.bytes().length == length) { decoder.readFixed(reusedFixed.bytes(), 0, length); return reusedFixed; } } byte[] bytes = new byte[length]; decoder.readFixed(bytes, 0, length); return new GenericData.Fixed(schema, bytes); } }
public Fixed(Schema schema) { bytes(new byte[schema.getFixedSize()]); } public Fixed(byte[] bytes) { bytes(bytes); }
public static float[] convertToFloat(GenericData.Fixed fixed) { ByteBuffer bytes = ByteBuffer.wrap(fixed.bytes()); // https://stackoverflow.com/questions/3770289/converting-array-of-primitives-to-array-of-containers-in-java // TODO use ArrayUtils? FloatBuffer asDoubleBuffer = bytes.asFloatBuffer(); float[] array = new float[asDoubleBuffer.capacity()]; asDoubleBuffer.get(array); return array; }
public static double[] convertToDouble(GenericData.Fixed fixed) { // Convert Fixed (which wraps a byte[]) to a double[] ByteBuffer bytes = ByteBuffer.wrap(fixed.bytes()); // https://stackoverflow.com/questions/3770289/converting-array-of-primitives-to-array-of-containers-in-java // TODO use ArrayUtils? DoubleBuffer asDoubleBuffer = bytes.asDoubleBuffer(); double[] array = new double[asDoubleBuffer.capacity()]; asDoubleBuffer.get(array); return array; }
@Override public void write(int repetitionLevel, Fixed buffer) { column.writeBinary(repetitionLevel, Binary.fromReusedByteArray(buffer.bytes())); } }
public static float[] convertToFloat(GenericData.Fixed fixed) { ByteBuffer bytes = ByteBuffer.wrap(fixed.bytes()); // https://stackoverflow.com/questions/3770289/converting-array-of-primitives-to-array-of-containers-in-java // TODO use ArrayUtils? FloatBuffer asDoubleBuffer = bytes.asFloatBuffer(); float[] array = new float[asDoubleBuffer.capacity()]; asDoubleBuffer.get(array); return array; }
@Override public Fixed read(Fixed reuse) { Fixed fixed; if (reuse != null) { fixed = reuse; } else { fixed = new Fixed(schema); } column.nextBinary().toByteBuffer().get(fixed.bytes()); return fixed; } }
public static double[] convertToDouble(GenericData.Fixed fixed) { // Convert Fixed (which wraps a byte[]) to a double[] ByteBuffer bytes = ByteBuffer.wrap(fixed.bytes()); // https://stackoverflow.com/questions/3770289/converting-array-of-primitives-to-array-of-containers-in-java // TODO use ArrayUtils? DoubleBuffer asDoubleBuffer = bytes.asDoubleBuffer(); double[] array = new double[asDoubleBuffer.capacity()]; asDoubleBuffer.get(array); return array; }