@Override public Iterable<AvroGenericRecordWritable> convertRecord(Schema outputSchema, GenericRecord inputRecord, WorkUnitState workUnit) throws DataConversionException { AvroGenericRecordWritable avroWritable = new AvroGenericRecordWritable(); avroWritable.setRecord(inputRecord); avroWritable.setFileSchema(outputSchema); avroWritable.setRecordReaderID(this.uid); return Lists.newArrayList(avroWritable); } }
public Writable serialize(Object o, ObjectInspector objectInspector, List<String> columnNames, List<TypeInfo> columnTypes, Schema schema) throws AvroSerdeException { StructObjectInspector soi = (StructObjectInspector) objectInspector; GenericData.Record record = new GenericData.Record(schema); List<? extends StructField> outputFieldRefs = soi.getAllStructFieldRefs(); if(outputFieldRefs.size() != columnNames.size()) { throw new AvroSerdeException("Number of input columns was different than output columns (in = " + columnNames.size() + " vs out = " + outputFieldRefs.size()); } int size = schema.getFields().size(); if(outputFieldRefs.size() != size) { throw new AvroSerdeException("Hive passed in a different number of fields than the schema expected: (Hive wanted " + outputFieldRefs.size() +", Avro expected " + schema.getFields().size()); } List<? extends StructField> allStructFieldRefs = soi.getAllStructFieldRefs(); List<Object> structFieldsDataAsList = soi.getStructFieldsDataAsList(o); for(int i = 0; i < size; i++) { Field field = schema.getFields().get(i); TypeInfo typeInfo = columnTypes.get(i); StructField structFieldRef = allStructFieldRefs.get(i); Object structFieldData = structFieldsDataAsList.get(i); ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector(); Object val = serialize(typeInfo, fieldOI, structFieldData, field.schema()); record.put(field.name(), val); } if(!GenericData.get().validate(schema, record)) { throw new SerializeToAvroException(schema, record); } cache.setRecord(record); return cache; }
@Override public AvroGenericRecordWritable getWritable(byte[] value) { GenericRecord avroRecord = null; try { avroRecord = dataReader.read(null, DecoderFactory.get().binaryDecoder(value, null)); } catch (IOException e) { Throwables.propagate(new SerDeException(e)); } avroGenericRecordWritable.setRecord(avroRecord); avroGenericRecordWritable.setRecordReaderID(uid); avroGenericRecordWritable.setFileSchema(avroRecord.getSchema()); return avroGenericRecordWritable; } }
public Writable serialize(Object o, ObjectInspector objectInspector, List<String> columnNames, List<TypeInfo> columnTypes, Schema schema) throws AvroSerdeException { StructObjectInspector soi = (StructObjectInspector) objectInspector; GenericData.Record record = new GenericData.Record(schema); List<? extends StructField> outputFieldRefs = soi.getAllStructFieldRefs(); if(outputFieldRefs.size() != columnNames.size()) throw new AvroSerdeException("Number of input columns was different than output columns (in = " + columnNames.size() + " vs out = " + outputFieldRefs.size()); int size = schema.getFields().size(); if(outputFieldRefs.size() != size) // Hive does this check for us, so we should be ok. throw new AvroSerdeException("Hive passed in a different number of fields than the schema expected: (Hive wanted " + outputFieldRefs.size() +", Avro expected " + schema.getFields().size()); List<? extends StructField> allStructFieldRefs = soi.getAllStructFieldRefs(); List<Object> structFieldsDataAsList = soi.getStructFieldsDataAsList(o); for(int i = 0; i < size; i++) { Field field = schema.getFields().get(i); TypeInfo typeInfo = columnTypes.get(i); StructField structFieldRef = allStructFieldRefs.get(i); Object structFieldData = structFieldsDataAsList.get(i); ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector(); Object val = serialize(typeInfo, fieldOI, structFieldData, field.schema()); record.put(field.name(), val); } if(!GenericData.get().validate(schema, record)) throw new SerializeToAvroException(schema, record); cache.setRecord(record); return cache; }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(isEmptyInput || !reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); record.setRecordReaderID(recordReaderID); record.setFileSchema(reader.getSchema()); return true; }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(!reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); return true; }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(isEmptyInput || !reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); record.setRecordReaderID(recordReaderID); record.setFileSchema(reader.getSchema()); return true; }
public Writable serialize(Object o, ObjectInspector objectInspector, List<String> columnNames, List<TypeInfo> columnTypes, Schema schema) throws AvroSerdeException { StructObjectInspector soi = (StructObjectInspector) objectInspector; GenericData.Record record = new GenericData.Record(schema); List<? extends StructField> outputFieldRefs = soi.getAllStructFieldRefs(); if(outputFieldRefs.size() != columnNames.size()) { throw new AvroSerdeException("Number of input columns was different than output columns (in = " + columnNames.size() + " vs out = " + outputFieldRefs.size()); } int size = schema.getFields().size(); if(outputFieldRefs.size() != size) { throw new AvroSerdeException("Hive passed in a different number of fields than the schema expected: (Hive wanted " + outputFieldRefs.size() +", Avro expected " + schema.getFields().size()); } List<? extends StructField> allStructFieldRefs = soi.getAllStructFieldRefs(); List<Object> structFieldsDataAsList = soi.getStructFieldsDataAsList(o); for(int i = 0; i < size; i++) { Field field = schema.getFields().get(i); TypeInfo typeInfo = columnTypes.get(i); StructField structFieldRef = allStructFieldRefs.get(i); Object structFieldData = structFieldsDataAsList.get(i); ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector(); Object val = serialize(typeInfo, fieldOI, structFieldData, field.schema()); record.put(field.name(), val); } if(!GenericData.get().validate(schema, record)) { throw new SerializeToAvroException(schema, record); } cache.setRecord(record); return cache; }
@Override public Iterable<AvroGenericRecordWritable> convertRecord(Schema outputSchema, GenericRecord inputRecord, WorkUnitState workUnit) throws DataConversionException { AvroGenericRecordWritable avroWritable = new AvroGenericRecordWritable(); avroWritable.setRecord(inputRecord); avroWritable.setFileSchema(outputSchema); avroWritable.setRecordReaderID(this.uid); return Lists.newArrayList(avroWritable); } }
public Writable serialize(Object o, ObjectInspector objectInspector, List<String> columnNames, List<TypeInfo> columnTypes, Schema schema) throws AvroSerdeException { StructObjectInspector soi = (StructObjectInspector) objectInspector; GenericData.Record record = new GenericData.Record(schema); List<? extends StructField> outputFieldRefs = soi.getAllStructFieldRefs(); if(outputFieldRefs.size() != columnNames.size()) { throw new AvroSerdeException("Number of input columns was different than output columns (in = " + columnNames.size() + " vs out = " + outputFieldRefs.size()); } int size = schema.getFields().size(); if(outputFieldRefs.size() != size) { throw new AvroSerdeException("Hive passed in a different number of fields than the schema expected: (Hive wanted " + outputFieldRefs.size() +", Avro expected " + schema.getFields().size()); } List<? extends StructField> allStructFieldRefs = soi.getAllStructFieldRefs(); List<Object> structFieldsDataAsList = soi.getStructFieldsDataAsList(o); for(int i = 0; i < size; i++) { Field field = schema.getFields().get(i); TypeInfo typeInfo = columnTypes.get(i); StructField structFieldRef = allStructFieldRefs.get(i); Object structFieldData = structFieldsDataAsList.get(i); ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector(); Object val = serialize(typeInfo, fieldOI, structFieldData, field.schema()); record.put(field.name(), val); } if(!GenericData.get().validate(schema, record)) { throw new SerializeToAvroException(schema, record); } cache.setRecord(record); return cache; }
@Override public AvroGenericRecordWritable getWritable(byte[] value) { GenericRecord avroRecord = null; try { avroRecord = dataReader.read(null, DecoderFactory.get().binaryDecoder(value, null)); } catch (IOException e) { Throwables.propagate(new SerDeException(e)); } avroGenericRecordWritable.setRecord(avroRecord); avroGenericRecordWritable.setRecordReaderID(uid); avroGenericRecordWritable.setFileSchema(avroRecord.getSchema()); return avroGenericRecordWritable; } }
public Writable serialize(Object o, ObjectInspector objectInspector, List<String> columnNames, List<TypeInfo> columnTypes, Schema schema) throws AvroSerdeException { StructObjectInspector soi = (StructObjectInspector) objectInspector; GenericData.Record record = new GenericData.Record(schema); List<? extends StructField> outputFieldRefs = soi.getAllStructFieldRefs(); if(outputFieldRefs.size() != columnNames.size()) { throw new AvroSerdeException("Number of input columns was different than output columns (in = " + columnNames.size() + " vs out = " + outputFieldRefs.size()); } int size = schema.getFields().size(); if(outputFieldRefs.size() != size) { throw new AvroSerdeException("Hive passed in a different number of fields than the schema expected: (Hive wanted " + outputFieldRefs.size() +", Avro expected " + schema.getFields().size()); } List<? extends StructField> allStructFieldRefs = soi.getAllStructFieldRefs(); List<Object> structFieldsDataAsList = soi.getStructFieldsDataAsList(o); for(int i = 0; i < size; i++) { Field field = schema.getFields().get(i); TypeInfo typeInfo = columnTypes.get(i); StructField structFieldRef = allStructFieldRefs.get(i); Object structFieldData = structFieldsDataAsList.get(i); ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector(); Object val = serialize(typeInfo, fieldOI, structFieldData, field.schema()); record.put(field.name(), val); } if(!GenericData.get().validate(schema, record)) { throw new SerializeToAvroException(schema, record); } cache.setRecord(record); return cache; }
@Override public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException { if(!reader.hasNext() || reader.pastSync(stop)) { return false; } GenericData.Record r = (GenericData.Record)reader.next(); record.setRecord(r); record.setRecordReaderID(recordReaderID); record.setFileSchema(reader.getSchema()); return true; }
public Writable serialize(Object o, ObjectInspector objectInspector, List<String> columnNames, List<TypeInfo> columnTypes, Schema schema) throws AvroSerdeException { StructObjectInspector soi = (StructObjectInspector) objectInspector; GenericData.Record record = new GenericData.Record(schema); List<? extends StructField> outputFieldRefs = soi.getAllStructFieldRefs(); if(outputFieldRefs.size() != columnNames.size()) { throw new AvroSerdeException("Number of input columns was different than output columns (in = " + columnNames.size() + " vs out = " + outputFieldRefs.size()); } int size = schema.getFields().size(); if(outputFieldRefs.size() != size) { throw new AvroSerdeException("Hive passed in a different number of fields than the schema expected: (Hive wanted " + outputFieldRefs.size() +", Avro expected " + schema.getFields().size()); } List<? extends StructField> allStructFieldRefs = soi.getAllStructFieldRefs(); List<Object> structFieldsDataAsList = soi.getStructFieldsDataAsList(o); for(int i = 0; i < size; i++) { Field field = schema.getFields().get(i); TypeInfo typeInfo = columnTypes.get(i); StructField structFieldRef = allStructFieldRefs.get(i); Object structFieldData = structFieldsDataAsList.get(i); ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector(); Object val = serialize(typeInfo, fieldOI, structFieldData, field.schema()); record.put(field.name(), val); } if(!GenericData.get().validate(schema, record)) { throw new SerializeToAvroException(schema, record); } cache.setRecord(record); return cache; }
@Override public Writable serialize(Object obj, ObjectInspector oi) throws SerDeException { StructObjectInspector soi = (StructObjectInspector) oi; List<Object> data = soi.getStructFieldsDataAsList(soi); StringObjectInspector foi = (StringObjectInspector) soi.getAllStructFieldRefs().get(0).getFieldObjectInspector(); try { agrw.setRecord(converter.convert(foi.getPrimitiveJavaObject(data.get(0)))); } catch (IOException e) { throw new SerDeException(e); } return agrw; } }
value.setRecord(wrapper.getRecord()); long decodeTime = ((secondTime - tempTime));