/** * Auto-generate the avro struct from schema * * @param schema schema for the avro struct * @param sb StringBuilder to hold the generated struct * @throws SerDeException if something goes wrong while generating the struct * */ private static void generateAvroStructFromSchema(Schema schema, StringBuilder sb) throws SerDeException { AvroObjectInspectorGenerator avig = new AvroObjectInspectorGenerator(schema); sb.append("struct<"); // Get the column names and their corresponding types List<String> columnNames = avig.getColumnNames(); List<TypeInfo> columnTypes = avig.getColumnTypes(); if (columnNames.size() != columnTypes.size()) { throw new AssertionError("The number of column names should be the same as column types"); } for (int i = 0; i < columnNames.size(); i++) { sb.append(columnNames.get(i)); sb.append(":"); sb.append(columnTypes.get(i).getTypeName()); sb.append(","); } trim(sb).append(">"); }
private void setColumnsIfPossible(WorkUnitState state) throws SerDeException { AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator( AvroSerdeUtils.determineSchemaOrReturnErrorSchema(state.getProperties())); List<String> columnNames = aoig.getColumnNames(); List<TypeInfo> columnTypes = aoig.getColumnTypes(); state.setProp(IOConstants.COLUMNS, StringUtils.join(columnNames, ",")); state.setProp(IOConstants.COLUMNS_TYPES, StringUtils.join(columnTypes, ",")); }
this.columnTypes = aoig.getColumnTypes(); this.oi = aoig.getObjectInspector();
oiGenerator = new AvroObjectInspectorGenerator(rs); deserializedObject = deserializer.deserialize(oiGenerator.getColumnNames(), oiGenerator.getColumnTypes(), avroWritable, rs); } catch (SerDeException se) {
/** * Auto-generate the avro struct from schema * * @param schema schema for the avro struct * @param sb StringBuilder to hold the generated struct * @throws SerDeException if something goes wrong while generating the struct * */ private static void generateAvroStructFromSchema(Schema schema, StringBuilder sb) throws SerDeException { AvroObjectInspectorGenerator avig = new AvroObjectInspectorGenerator(schema); sb.append("struct<"); // Get the column names and their corresponding types List<String> columnNames = avig.getColumnNames(); List<TypeInfo> columnTypes = avig.getColumnTypes(); if (columnNames.size() != columnTypes.size()) { throw new AssertionError("The number of column names should be the same as column types"); } for (int i = 0; i < columnNames.size(); i++) { sb.append(columnNames.get(i)); sb.append(":"); sb.append(columnTypes.get(i).getTypeName()); sb.append(","); } trim(sb).append(">"); }
@Override public void initialize(Configuration configuration, Properties properties) throws SerDeException { // Reset member variables so we don't get in a half-constructed state if(schema != null) LOG.info("Resetting already initialized AvroSerDe"); schema = null; oi = null; columnNames = null; columnTypes = null; schema = AvroSerdeUtils.determineSchemaOrReturnErrorSchema(properties); if(configuration == null) { LOG.info("Configuration null, not inserting schema"); } else { // force output files to have a .avro extension configuration.set("hive.output.file.extension", ".avro"); configuration.set(AvroSerdeUtils.AVRO_SERDE_SCHEMA, schema.toString(false)); } badSchema = schema.equals(SchemaResolutionProblem.SIGNAL_BAD_SCHEMA); AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(schema); this.columnNames = aoig.getColumnNames(); this.columnTypes = aoig.getColumnTypes(); this.oi = aoig.getObjectInspector(); }
private void setColumnsIfPossible(WorkUnitState state) throws SerDeException { AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator( AvroSerdeUtils.determineSchemaOrReturnErrorSchema(state.getProperties())); List<String> columnNames = aoig.getColumnNames(); List<TypeInfo> columnTypes = aoig.getColumnTypes(); state.setProp(IOConstants.COLUMNS, StringUtils.join(columnNames, ",")); state.setProp(IOConstants.COLUMNS_TYPES, StringUtils.join(columnTypes, ",")); }
this.columnTypes = aoig.getColumnTypes(); this.oi = aoig.getObjectInspector();
this.columnTypes = aoig.getColumnTypes(); this.oi = aoig.getObjectInspector();
this.columnTypes = aoig.getColumnTypes(); this.oi = aoig.getObjectInspector();
this.columnTypes = aoig.getColumnTypes(); this.oi = aoig.getObjectInspector();
oiGenerator = new AvroObjectInspectorGenerator(rs); deserializedObject = deserializer.deserialize(oiGenerator.getColumnNames(), oiGenerator.getColumnTypes(), avroWritable, rs); } catch (SerDeException se) {
oiGenerator = new AvroObjectInspectorGenerator(rs); deserializedObject = deserializer.deserialize(oiGenerator.getColumnNames(), oiGenerator.getColumnTypes(), avroWritable, rs); } catch (SerDeException se) {
oiGenerator = new AvroObjectInspectorGenerator(rs); deserializedObject = deserializer.deserialize(oiGenerator.getColumnNames(), oiGenerator.getColumnTypes(), avroWritable, rs); } catch (SerDeException se) {
oiGenerator = new AvroObjectInspectorGenerator(rs); deserializedObject = deserializer.deserialize(oiGenerator.getColumnNames(), oiGenerator.getColumnTypes(), avroWritable, rs); } catch (SerDeException se) {