reader.getCurrentRow(cols); cols.resetValid(8); Object row = serDe.deserialize(cols); .getObjectInspector(); List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs(); assertEquals("Field size should be 8", 8, fieldRefs.size()); BytesRefArrayWritable.class, serDe.getSerializedClass()); BytesRefArrayWritable serializedBytes = (BytesRefArrayWritable) serDe .serialize(row, oi); assertEquals("Serialized data", patialS, serializedBytes);
@Override @SuppressWarnings("deprecation") public Serializer createSerializer() { try { ColumnarSerDe columnarSerDe = new ColumnarSerDe(); Properties tableProperties = new Properties(); tableProperties.setProperty("columns", "test"); tableProperties.setProperty("columns.types", "string"); columnarSerDe.initialize(new JobConf(false), tableProperties); return columnarSerDe; } catch (SerDeException e) { throw new RuntimeException(e); } }
reader.getCurrentRow(cols); cols.resetValid(8); Object row = serDe.deserialize(cols); .getObjectInspector(); List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs(); assertEquals("Field size should be 8", 8, fieldRefs.size());
List<Object> list = soi.getStructFieldsDataAsList(obj); List<? extends StructField> declaredFields = (serdeParams.getRowTypeInfo() != null && ((StructTypeInfo) serdeParams .getRowTypeInfo()).getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()) .getAllStructFieldRefs() : null;
Path file = null; AbstractSerDe serDe = new ColumnarSerDe();
ColumnarSerDe serDe = new ColumnarSerDe(); serDe.initialize(conf, tbl); final ColumnarStruct row = (ColumnarStruct) serDe.deserialize(cols); final ArrayList<Object> objects = row.getFieldsAsList(); for (final Object object : objects) {
@Override public Object deserialize(Writable blob) throws SerDeException { // Ideally this should throw UnsupportedOperationException as the serde is // vectorized serde. But since RC file reader does not support vectorized reading this // is left as it is. This function will be called from VectorizedRowBatchCtx::addRowToBatch // to deserialize the row one by one and populate the batch. Once RC file reader supports vectorized // reading this serde and be standalone serde with no dependency on ColumnarSerDe. return super.deserialize(blob); }
List<Object> list = soi.getStructFieldsDataAsList(obj); List<? extends StructField> declaredFields = (serdeParams.getRowTypeInfo() != null && ((StructTypeInfo) serdeParams .getRowTypeInfo()).getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()) .getAllStructFieldRefs() : null;
deserializer = new ColumnarSerDe();
@Override public Serializer createSerializer() { try { ColumnarSerDe columnarSerDe = new ColumnarSerDe(); Properties tableProperties = new Properties(); tableProperties.setProperty("columns", "test"); tableProperties.setProperty("columns.types", "string"); columnarSerDe.initialize(new JobConf(false), tableProperties); return columnarSerDe; } catch (SerDeException e) { throw new RuntimeException(e); } }
reader.getCurrentRow(cols); cols.resetValid(8); Object row = serDe.deserialize(cols); .getObjectInspector(); List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs(); assertEquals("Field size should be 8", 8, fieldRefs.size()); BytesRefArrayWritable.class, serDe.getSerializedClass()); BytesRefArrayWritable serializedText = (BytesRefArrayWritable) serDe .serialize(row, oi); assertEquals("Serialized data", s, serializedText); actualRead++;
List<Object> list = soi.getStructFieldsDataAsList(obj); List<? extends StructField> declaredFields = (serdeParams.getRowTypeInfo() != null && ((StructTypeInfo) serdeParams .getRowTypeInfo()).getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()) .getAllStructFieldRefs() : null;
cleanup(); serDe = new ColumnarSerDe();
@Override @SuppressWarnings("deprecation") public Serializer createSerializer() { try { ColumnarSerDe columnarSerDe = new ColumnarSerDe(); Properties tableProperties = new Properties(); tableProperties.setProperty("columns", "test"); tableProperties.setProperty("columns.types", "string"); columnarSerDe.initialize(new JobConf(false), tableProperties); return columnarSerDe; } catch (SerDeException e) { throw new RuntimeException(e); } }
List<Object> list = soi.getStructFieldsDataAsList(obj); List<? extends StructField> declaredFields = (serdeParams.getRowTypeInfo() != null && ((StructTypeInfo) serdeParams .getRowTypeInfo()).getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()) .getAllStructFieldRefs() : null;
@Test(enabled = false) public void testRcTextPageSource() throws Exception { HiveOutputFormat<?, ?> outputFormat = new RCFileOutputFormat(); InputFormat<?, ?> inputFormat = new RCFileInputFormat<>(); @SuppressWarnings("deprecation") SerDe serde = new ColumnarSerDe(); File file = File.createTempFile("presto_test", "rc-binary"); file.delete(); try { FileSplit split = createTestFile(file.getAbsolutePath(), outputFormat, serde, null, TEST_COLUMNS, NUM_ROWS); testPageSourceFactory(new RcFilePageSourceFactory(TYPE_MANAGER), split, inputFormat, serde, TEST_COLUMNS); } finally { //noinspection ResultOfMethodCallIgnored file.delete(); } }
@Override public Serializer createSerializer() { try { ColumnarSerDe columnarSerDe = new ColumnarSerDe(); Properties tableProperties = new Properties(); tableProperties.setProperty("columns", "test"); tableProperties.setProperty("columns.types", "string"); columnarSerDe.initialize(new JobConf(false), tableProperties); return columnarSerDe; } catch (SerDeException e) { throw new RuntimeException(e); } }
List<Object> list = soi.getStructFieldsDataAsList(obj); List<? extends StructField> declaredFields = (serdeParams.getRowTypeInfo() != null && ((StructTypeInfo) serdeParams .getRowTypeInfo()).getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()) .getAllStructFieldRefs() : null;
@Test public void testRCText() throws Exception { List<TestColumn> testColumns = ImmutableList.copyOf(filter(TEST_COLUMNS, testColumn -> { return !testColumn.getName().equals("t_struct_null") // TODO: This is a bug in the RC text reader && !testColumn.getName().equals("t_map_null_key_complex_key_value"); // RC file does not support complex type as key of a map })); HiveOutputFormat<?, ?> outputFormat = new RCFileOutputFormat(); InputFormat<?, ?> inputFormat = new RCFileInputFormat<>(); @SuppressWarnings("deprecation") SerDe serde = new ColumnarSerDe(); File file = File.createTempFile("presto_test", "rc-text"); try { FileSplit split = createTestFile(file.getAbsolutePath(), outputFormat, serde, null, testColumns, NUM_ROWS); testCursorProvider(new ColumnarTextHiveRecordCursorProvider(), split, inputFormat, serde, testColumns, NUM_ROWS); testCursorProvider(new GenericHiveRecordCursorProvider(), split, inputFormat, serde, testColumns, NUM_ROWS); } finally { //noinspection ResultOfMethodCallIgnored file.delete(); } }
List<Object> list = soi.getStructFieldsDataAsList(obj); List<? extends StructField> declaredFields = (serdeParams.getRowTypeInfo() != null && ((StructTypeInfo) serdeParams .getRowTypeInfo()).getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()) .getAllStructFieldRefs() : null;