public void setSchema(ResourceSchema schema) { schemaField = new ResourceFieldSchema(); schemaField.setType(DataType.TUPLE); try { schemaField.setSchema(schema); } catch (IOException ex) { throw new EsHadoopIllegalStateException(String.format("Cannot use schema [%s]", schema), ex); } }
bagSubFieldSchemas[0] = new ResourceFieldSchema().setName(innerTupleName) .setDescription("The tuple in the bag") .setType(DataType.TUPLE); HCatFieldSchema arrayElementFieldSchema = hfs.getArrayElementSchema().get(0); if (arrayElementFieldSchema.getType() == Type.STRUCT) { innerTupleFieldSchemas[0] = new ResourceFieldSchema().setName(innerFieldName) .setDescription("The inner field in the tuple in the bag") .setType(getPigType(arrayElementFieldSchema))
byte pigType = TypeUtil.getPigDataTypeForPhoenixType(phoenixDataType); ResourceFieldSchema field = new ResourceFieldSchema(); field.setType(pigType).setName(cinfo.getDisplayName()); fields[i++] = field;
private static ResourceFieldSchema getResourceSchemaFromFieldSchema(HCatFieldSchema hfs) throws IOException { ResourceFieldSchema rfSchema; // if we are dealing with a bag or tuple column - need to worry about subschema if (hfs.getType() == Type.STRUCT) { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) .setDescription(hfs.getComment()) .setType(getPigType(hfs)) .setSchema(getTupleSubSchema(hfs)); } else if (hfs.getType() == Type.ARRAY) { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) .setDescription(hfs.getComment()) .setType(getPigType(hfs)) .setSchema(getBagSubSchema(hfs)); } else { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) .setDescription(hfs.getComment()) .setType(getPigType(hfs)) .setSchema(null); // no munging inner-schemas } return rfSchema; }
private void init(Schema inputSchema, GenericUDF evalUDF, ConstantObjectInspectInfo constantsInfo) throws IOException { ResourceSchema rs = new ResourceSchema(inputSchema); ResourceFieldSchema wrappedTupleFieldSchema = new ResourceFieldSchema(); wrappedTupleFieldSchema.setType(DataType.TUPLE); wrappedTupleFieldSchema.setSchema(rs); TypeInfo ti = HiveUtils.getTypeInfo(wrappedTupleFieldSchema); inputObjectInspector = (StructObjectInspector)HiveUtils.createObjectInspector(ti); try { ObjectInspector[] arguments = new ObjectInspector[inputSchema.size()]; for (int i=0;i<inputSchema.size();i++) { if (constantsInfo!=null && !constantsInfo.isEmpty() && constantsInfo.get(i)!=null) { arguments[i] = constantsInfo.get(i); } else { arguments[i] = inputObjectInspector.getAllStructFieldRefs().get(i).getFieldObjectInspector(); } } outputObjectInspector = evalUDF.initialize(arguments); } catch (Exception e) { throw new IOException(e); } } }
@Override public Schema outputSchema(Schema input) { try { if (!inited) { schemaInfo.init(getInputSchema(), udtf, constantsInfo); inited = true; } ResourceFieldSchema rfs = HiveUtils.getResourceFieldSchema( TypeInfoUtils.getTypeInfoFromObjectInspector(schemaInfo.outputObjectInspector)); ResourceSchema tupleSchema = new ResourceSchema(); tupleSchema.setFields(new ResourceFieldSchema[] {rfs}); ResourceFieldSchema bagFieldSchema = new ResourceFieldSchema(); bagFieldSchema.setType(DataType.BAG); bagFieldSchema.setSchema(tupleSchema); ResourceSchema bagSchema = new ResourceSchema(); bagSchema.setFields(new ResourceFieldSchema[] {bagFieldSchema}); return Schema.getPigSchema(bagSchema); } catch (Exception e) { throw new RuntimeException(e); } }
private void init(Schema inputSchema, GenericUDTF udtf, ConstantObjectInspectInfo constantsInfo) throws IOException { ResourceSchema rs = new ResourceSchema(inputSchema); ResourceFieldSchema wrappedTupleFieldSchema = new ResourceFieldSchema(); wrappedTupleFieldSchema.setType(DataType.TUPLE); wrappedTupleFieldSchema.setSchema(rs); TypeInfo ti = HiveUtils.getTypeInfo(wrappedTupleFieldSchema); inputObjectInspector = (StructObjectInspector)HiveUtils.createObjectInspector(ti); if (constantsInfo!=null) { constantsInfo.injectConstantObjectInspector(inputObjectInspector); } try { outputObjectInspector = udtf.initialize(inputObjectInspector); } catch (Exception e) { throw new IOException(e); } } }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema(); schema.setType(DataType.CHARARRAY); return schema; }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema(); schema.setType(DataType.NULL); return schema; }
public ResourceSchema fixSchema(ResourceSchema s){ for (ResourceFieldSchema filed : s.getFields()) { if(filed.getType() == DataType.NULL) filed.setType(DataType.BYTEARRAY); } return s; }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema(); schema.setType(DataType.BYTEARRAY); return schema; }
public void setSchema(ResourceSchema schema) { schemaField = new ResourceFieldSchema(); schemaField.setType(DataType.TUPLE); try { schemaField.setSchema(schema); } catch (IOException ex) { throw new EsHadoopIllegalStateException(String.format("Cannot use schema [%s]", schema), ex); } }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema(); schema.setType(DataType.LONG); return schema; }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema(); schema.setType(DataType.BYTEARRAY); return schema; }
@Override public ResourceFieldSchema getLoadSchema() throws IOException { ResourceFieldSchema schema = new ResourceFieldSchema(); schema.setType(DataType.INTEGER); return schema; }
private static ResourceFieldSchema convert(Type type) throws IOException { ResourceFieldSchema result = new ResourceFieldSchema(); result.setType(convertType(type)); if (!type.isPrimitiveType()) { result.setSchema(convertComplex(type)); } return result; }
@Override public void checkSchema(ResourceSchema rs) throws IOException { ResourceFieldSchema fs = new ResourceFieldSchema(); fs.setType(DataType.TUPLE); fs.setSchema(rs); typeInfo = HiveUtils.getTypeInfo(fs); Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass()); p.setProperty(signature + SchemaSignatureSuffix, ObjectSerializer.serialize(typeInfo)); }
when(record.getResultMap()).thenReturn(values); ResourceFieldSchema field = new ResourceFieldSchema().setType(DataType.CHARARRAY); ResourceFieldSchema field1 = new ResourceFieldSchema().setType(DataType.INTEGER); ResourceFieldSchema field2 = new ResourceFieldSchema().setType(DataType.LONG); ResourceFieldSchema field3 = new ResourceFieldSchema().setType(DataType.BYTEARRAY); ResourceFieldSchema field4 = new ResourceFieldSchema().setType(DataType.TUPLE); ResourceFieldSchema[] projectedColumns = { field, field1, field2, field3 , field4 }; assertEquals(2,doubleArrayTuple.size()); field = new ResourceFieldSchema().setType(DataType.BIGDECIMAL); field1 = new ResourceFieldSchema().setType(DataType.BIGINTEGER); values.clear(); values.put("first", new BigDecimal(123123123.123213));
@Test public void testSchema() throws SQLException, IOException { final Configuration configuration = mock(Configuration.class); when(configuration.get(PhoenixConfigurationUtil.SCHEMA_TYPE)).thenReturn(SchemaType.TABLE.name()); final ResourceSchema actual = PhoenixPigSchemaUtil.getResourceSchema( configuration, new Dependencies() { List<ColumnInfo> getSelectColumnMetadataList( Configuration configuration) throws SQLException { return Lists.newArrayList(ID_COLUMN, NAME_COLUMN); } }); // expected schema. final ResourceFieldSchema[] fields = new ResourceFieldSchema[2]; fields[0] = new ResourceFieldSchema().setName("ID") .setType(DataType.LONG); fields[1] = new ResourceFieldSchema().setName("NAME") .setType(DataType.CHARARRAY); final ResourceSchema expected = new ResourceSchema().setFields(fields); assertEquals(expected.toString(), actual.toString()); }
private static ResourceFieldSchema getResourceSchemaFromFieldSchema(HCatFieldSchema hfs) throws IOException { ResourceFieldSchema rfSchema; // if we are dealing with a bag or tuple column - need to worry about subschema if (hfs.getType() == Type.STRUCT) { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) .setDescription(hfs.getComment()) .setType(getPigType(hfs)) .setSchema(getTupleSubSchema(hfs)); } else if (hfs.getType() == Type.ARRAY) { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) .setDescription(hfs.getComment()) .setType(getPigType(hfs)) .setSchema(getBagSubSchema(hfs)); } else { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) .setDescription(hfs.getComment()) .setType(getPigType(hfs)) .setSchema(null); // no munging inner-schemas } return rfSchema; }