public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException { // Store the schema in a side file in the same directory. MapReduce // does not include files starting with "_" when reading data for a job. JsonMetadata metadataWriter = new JsonMetadata(); byte recordDel = '\n'; byte fieldDel = '\t'; metadataWriter.setFieldDel(fieldDel); metadataWriter.setRecordDel(recordDel); metadataWriter.storeSchema(schema, location, job); }
@Override public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException { if (isSchemaOn) { JsonMetadata metadataWriter = new JsonMetadata(); byte recordDel = '\n'; metadataWriter.setFieldDel(fieldDel); metadataWriter.setRecordDel(recordDel); metadataWriter.storeSchema(schema, location, job); } }
@Override public ResourceSchema getSchema(String location, Job job) throws IOException { ResourceSchema s; if (schema!=null) { s = schema; } else { // Parse the schema s = (new JsonMetadata()).getSchema(location, job, true); if (s == null) { throw new IOException("Unable to parse schema found in file in " + location); } } // Now that we have determined the schema, store it in our // UDFContext properties object so we have it when we need it on the // backend UDFContext udfc = UDFContext.getUDFContext(); Properties p = udfc.getUDFProperties(this.getClass(), new String[]{udfcSignature}); p.setProperty(SCHEMA_SIGNATURE, s.toString()); return s; }
@Override public ResourceSchema getSchema(String location, Job job) throws IOException { if (!dontLoadSchema) { schema = (new JsonMetadata()).getSchema(location, job, isSchemaOn); if (signature != null && schema != null) { if(tagFile) { schema = Utils.getSchemaWithInputSourceTag(schema, "INPUT_FILE_NAME"); } else if(tagPath) { schema = Utils.getSchemaWithInputSourceTag(schema, "INPUT_FILE_PATH"); } Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass(), new String[] {signature}); p.setProperty(signature + ".schema", schema.toString()); } } return schema; }