@Override public ResourceSchema getSchema(String location, Job job) throws IOException { if(schema != null) { return schema; } PhoenixConfigurationUtil.loadHBaseConfiguration(job); final Configuration configuration = job.getConfiguration(); this.initializePhoenixPigConfiguration(location, configuration); this.schema = PhoenixPigSchemaUtil.getResourceSchema(this.config); if(LOG.isDebugEnabled()) { LOG.debug(String.format("Resource Schema generated for location [%s] is [%s]", location, schema.toString())); } this.storeInUDFContext(this.contextSignature, RESOURCE_SCHEMA_SIGNATURE, ObjectSerializer.serialize(schema)); return schema; }
@Test public void testSchema() throws SQLException, IOException { final Configuration configuration = mock(Configuration.class); when(configuration.get(PhoenixConfigurationUtil.SCHEMA_TYPE)).thenReturn(SchemaType.TABLE.name()); final ResourceSchema actual = PhoenixPigSchemaUtil.getResourceSchema( configuration, new Dependencies() { List<ColumnInfo> getSelectColumnMetadataList( Configuration configuration) throws SQLException { return Lists.newArrayList(ID_COLUMN, NAME_COLUMN); } }); // expected schema. final ResourceFieldSchema[] fields = new ResourceFieldSchema[2]; fields[0] = new ResourceFieldSchema().setName("ID") .setType(DataType.LONG); fields[1] = new ResourceFieldSchema().setName("NAME") .setType(DataType.CHARARRAY); final ResourceSchema expected = new ResourceSchema().setFields(fields); assertEquals(expected.toString(), actual.toString()); }
/** * {@inheritDoc} */ @Override public void checkSchema(ResourceSchema s) throws IOException { getProperties().setProperty(SCHEMA, s.toString()); }
@Override public void checkSchema(ResourceSchema s) throws IOException { // We won't really check the schema here, we'll store it in our // UDFContext properties object so we have it when we need it on the // backend UDFContext udfc = UDFContext.getUDFContext(); Properties p = udfc.getUDFProperties(this.getClass(), new String[]{udfcSignature}); p.setProperty(SCHEMA_SIGNATURE, fixSchema(s).toString()); }
@Override public ResourceSchema getSchema(String location, Job job) throws IOException { if(schema != null) { return schema; } PhoenixConfigurationUtil.loadHBaseConfiguration(job); final Configuration configuration = job.getConfiguration(); this.initializePhoenixPigConfiguration(location, configuration); this.schema = PhoenixPigSchemaUtil.getResourceSchema(this.config); if(LOG.isDebugEnabled()) { LOG.debug(String.format("Resource Schema generated for location [%s] is [%s]", location, schema.toString())); } this.storeInUDFContext(this.contextSignature, RESOURCE_SCHEMA_SIGNATURE, ObjectSerializer.serialize(schema)); return schema; }
@Override public ResourceSchema getSchema(String location, Job job) throws IOException { ResourceSchema s; if (schema!=null) { s = schema; } else { // Parse the schema s = (new JsonMetadata()).getSchema(location, job, true); if (s == null) { throw new IOException("Unable to parse schema found in file in " + location); } } // Now that we have determined the schema, store it in our // UDFContext properties object so we have it when we need it on the // backend UDFContext udfc = UDFContext.getUDFContext(); Properties p = udfc.getUDFProperties(this.getClass(), new String[]{udfcSignature}); p.setProperty(SCHEMA_SIGNATURE, s.toString()); return s; }
private static void convertToPigSchema(Schema icebergSchema, String expectedPigSchema, String assertMessage) throws IOException { ResourceSchema pigSchema = SchemaUtil.convert(icebergSchema); assertEquals(assertMessage, expectedPigSchema, pigSchema.toString()); } }
"Don't know how to encode type " + DataType.findTypeName(type) + " in schema " + ((schema == null) ? "" : schema.toString()) + "\n");
@Test public void testTupleInMap() throws IOException { Schema icebergSchema = new Schema( optional( 1, "nested_list", MapType.ofOptional( 2, 3, StringType.get(), ListType.ofOptional( 4, StructType.of( required(5, "id", LongType.get()), optional(6, "data", StringType.get())))))); ResourceSchema pigSchema = SchemaUtil.convert(icebergSchema); assertEquals("nested_list:[{(id:long,data:chararray)}]", pigSchema.toString()); // The output should contain a nested struct within a list within a map, I think. }
@Test public void testPrimitive() throws IOException { Schema icebergSchema = new Schema( optional(1, "b", BooleanType.get()), optional(1, "i", IntegerType.get()), optional(2, "l", LongType.get()), optional(3, "f", FloatType.get()), optional(4, "d", DoubleType.get()), optional(5, "dec", DecimalType.of(0,2)), optional(5, "s", StringType.get()), optional(6,"bi", BinaryType.get()) ); ResourceSchema pigSchema = SchemaUtil.convert(icebergSchema); assertEquals("b:boolean,i:int,l:long,f:float,d:double,dec:bigdecimal,s:chararray,bi:bytearray", pigSchema.toString()); }