/** * It writes a record to Parquet. * @param record Contains the record that is going to be written. */ public void write(final ParquetHiveRecord record) { if (record != null) { if (messageWriter == null) { try { messageWriter = createMessageWriter(record.getObjectInspector(), schema); } catch (RuntimeException e) { String errorMessage = "Parquet record is malformed: " + e.getMessage(); LOG.error(errorMessage, e); throw new RuntimeException(errorMessage, e); } } messageWriter.write(record.getObject()); } }
public ParquetHiveSerDe() { parquetRow = new ParquetHiveRecord(); stats = new SerDeStats(); }
private void deserializeAndSerializeLazySimple(final ParquetHiveSerDe serDe, final ArrayWritable t) throws SerDeException { // Get the row structure final StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector(); // Deserialize final Object row = serDe.deserialize(t); assertEquals("deserialization gives the wrong object class", row.getClass(), ArrayWritable.class); assertEquals("size correct after deserialization", serDe.getSerDeStats().getRawDataSize(), t.get().length); assertEquals("deserialization gives the wrong object", t, row); // Serialize final ParquetHiveRecord serializedArr = (ParquetHiveRecord) serDe.serialize(row, oi); assertEquals("size correct after serialization", serDe.getSerDeStats().getRawDataSize(), ((ArrayWritable)serializedArr.getObject()).get().length); assertTrue("serialized object should be equal to starting object", arrayWritableEquals(t, (ArrayWritable)serializedArr.getObject())); }
/** * It writes a record to Parquet. * @param record Contains the record that is going to be written. */ public void write(final ParquetHiveRecord record) { if (record != null) { if (messageWriter == null) { try { messageWriter = createMessageWriter(record.getObjectInspector(), schema); } catch (RuntimeException e) { String errorMessage = "Parquet record is malformed: " + e.getMessage(); LOG.error(errorMessage, e); throw new RuntimeException(errorMessage, e); } } messageWriter.write(record.getObject()); } }
public ParquetHiveSerDe() { parquetRow = new ParquetHiveRecord(); stats = new SerDeStats(); }
/** * It writes all record values to the Parquet RecordConsumer. * * @param record Contains the record that are going to be written. */ public void write(final ParquetHiveRecord record) { if (record != null) { recordConsumer.startMessage(); try { writeGroupFields(record.getObject(), record.getObjectInspector(), schema); } catch (RuntimeException e) { String errorMessage = "Parquet record is malformed: " + e.getMessage(); log.error(errorMessage, e); throw new RuntimeException(errorMessage, e); } recordConsumer.endMessage(); } }
private ParquetHiveRecord getParquetWritable(String columnNames, String columnTypes, ArrayWritable record) throws SerDeException { Properties recordProperties = new Properties(); recordProperties.setProperty("columns", columnNames); recordProperties.setProperty("columns.types", columnTypes); ParquetHiveSerDe serDe = new ParquetHiveSerDe(); SerDeUtils.initializeSerDe(serDe, new Configuration(), recordProperties, null); return new ParquetHiveRecord(serDe.deserialize(record), getObjectInspector(columnNames, columnTypes)); }
/** * It writes all record values to the Parquet RecordConsumer. * @param record Contains the record that are going to be written. */ public void write(final ParquetHiveRecord record) { if (record != null) { recordConsumer.startMessage(); try { writeGroupFields(record.getObject(), record.getObjectInspector(), schema); } catch (RuntimeException e) { String errorMessage = "Parquet record is malformed: " + e.getMessage(); LOG.error(errorMessage, e); throw new RuntimeException(errorMessage, e); } recordConsumer.endMessage(); } }
public ParquetHiveSerDe() { parquetRow = new ParquetHiveRecord(); stats = new SerDeStats(); }
/** * It writes all record values to the Parquet RecordConsumer. * * @param record Contains the record that are going to be written. */ public void write(final ParquetHiveRecord record) { if (record != null) { recordConsumer.startMessage(); try { writeGroupFields(record.getObject(), record.getObjectInspector(), schema); } catch (RuntimeException e) { String errorMessage = "Parquet record is malformed: " + e.getMessage(); log.error(errorMessage, e); throw new RuntimeException(errorMessage, e); } recordConsumer.endMessage(); } }
/** * It writes all record values to the Parquet RecordConsumer. * * @param record Contains the record that are going to be written. */ public void write(final ParquetHiveRecord record) { if (record != null) { recordConsumer.startMessage(); try { writeGroupFields(record.getObject(), record.getObjectInspector(), schema); } catch (RuntimeException e) { String errorMessage = "Parquet record is malformed: " + e.getMessage(); log.error(errorMessage, e); throw new RuntimeException(errorMessage, e); } recordConsumer.endMessage(); } }
/** * It writes all record values to the Parquet RecordConsumer. * * @param record Contains the record that are going to be written. */ public void write(final ParquetHiveRecord record) { if (record != null) { recordConsumer.startMessage(); try { writeGroupFields(record.getObject(), record.getObjectInspector(), schema); } catch (RuntimeException e) { String errorMessage = "Parquet record is malformed: " + e.getMessage(); log.error(errorMessage, e); throw new RuntimeException(errorMessage, e); } recordConsumer.endMessage(); } }