/** * Gets the data schema. * @return the table schema */ public HCatSchema getDataSchema() { return this.partitionInfo.getPartitionSchema(); }
private void createDeserializer(HCatSplit hcatSplit, HiveStorageHandler storageHandler, TaskAttemptContext taskContext) throws IOException { deserializer = ReflectionUtils.newInstance(storageHandler.getSerDeClass(), taskContext.getConfiguration()); try { InternalUtil.initializeDeserializer(deserializer, storageHandler.getConf(), hcatSplit.getPartitionInfo().getTableInfo(), hcatSplit.getPartitionInfo().getPartitionSchema()); } catch (SerDeException e) { throw new IOException("Failed initializing deserializer " + storageHandler.getSerDeClass().getName(), e); } }
/** * gets values for fields requested by output schema which will not be in the data */ private static Map<String, Object> getColValsNotInDataColumns(HCatSchema outputSchema, PartInfo partInfo) throws HCatException { HCatSchema dataSchema = partInfo.getPartitionSchema(); Map<String, Object> vals = new HashMap<String, Object>(); for (String fieldName : outputSchema.getFieldNames()) { if (dataSchema.getPosition(fieldName) == null) { // this entry of output is not present in the output schema // so, we first check the table schema to see if it is a part col if (partInfo.getPartitionValues().containsKey(fieldName)) { // First, get the appropriate field schema for this field HCatFieldSchema fschema = outputSchema.get(fieldName); // For a partition key type, this will be a primitive typeinfo. // Obtain relevant object inspector for this typeinfo ObjectInspector oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(fschema.getTypeInfo()); // get appropriate object from the string representation of the value in partInfo.getPartitionValues() // Essentially, partition values are represented as strings, but we want the actual object type associated Object objVal = ObjectInspectorConverters .getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi) .convert(partInfo.getPartitionValues().get(fieldName)); vals.put(fieldName, objVal); } else { vals.put(fieldName, null); } } } return vals; }
/** * Gets the data schema. * @return the table schema */ public HCatSchema getDataSchema() { return this.partitionInfo.getPartitionSchema(); }
/** * Gets the data schema. * @return the table schema */ public HCatSchema getDataSchema() { return this.partitionInfo.getPartitionSchema(); }
/** * Gets the data schema. * @return the table schema */ public HCatSchema getDataSchema() { return this.partitionInfo.getPartitionSchema(); }
/** * Gets the data schema. * @return the table schema */ public HCatSchema getDataSchema() { return this.partitionInfo.getPartitionSchema(); }
private void createDeserializer(HCatSplit hcatSplit, HiveStorageHandler storageHandler, TaskAttemptContext taskContext) throws IOException { deserializer = ReflectionUtils.newInstance(storageHandler.getSerDeClass(), taskContext.getConfiguration()); try { InternalUtil.initializeDeserializer(deserializer, storageHandler.getConf(), hcatSplit.getPartitionInfo().getTableInfo(), hcatSplit.getPartitionInfo().getPartitionSchema()); } catch (SerDeException e) { throw new IOException("Failed initializing deserializer " + storageHandler.getSerDeClass().getName(), e); } }
private void createDeserializer(HCatSplit hcatSplit, HiveStorageHandler storageHandler, TaskAttemptContext taskContext) throws IOException { deserializer = ReflectionUtils.newInstance(storageHandler.getSerDeClass(), taskContext.getConfiguration()); try { InternalUtil.initializeDeserializer(deserializer, storageHandler.getConf(), hcatSplit.getPartitionInfo().getTableInfo(), hcatSplit.getPartitionInfo().getPartitionSchema()); } catch (SerDeException e) { throw new IOException("Failed initializing deserializer " + storageHandler.getSerDeClass().getName(), e); } }
private void createDeserializer(HCatSplit hcatSplit, HiveStorageHandler storageHandler, TaskAttemptContext taskContext) throws IOException { deserializer = ReflectionUtils.newInstance(storageHandler.getSerDeClass(), taskContext.getConfiguration()); try { InternalUtil.initializeDeserializer(deserializer, storageHandler.getConf(), hcatSplit.getPartitionInfo().getTableInfo(), hcatSplit.getPartitionInfo().getPartitionSchema()); } catch (SerDeException e) { throw new IOException("Failed initializing deserializer " + storageHandler.getSerDeClass().getName(), e); } }
private void createDeserializer(HCatSplit hcatSplit, HiveStorageHandler storageHandler, TaskAttemptContext taskContext) throws IOException { deserializer = ReflectionUtils.newInstance(storageHandler.getSerDeClass(), taskContext.getConfiguration()); try { InternalUtil.initializeDeserializer(deserializer, storageHandler.getConf(), hcatSplit.getPartitionInfo().getTableInfo(), hcatSplit.getPartitionInfo().getPartitionSchema()); } catch (SerDeException e) { throw new IOException("Failed initializing deserializer " + storageHandler.getSerDeClass().getName(), e); } }
/** * gets values for fields requested by output schema which will not be in the data */ private static Map<String, Object> getColValsNotInDataColumns(HCatSchema outputSchema, PartInfo partInfo) throws HCatException { HCatSchema dataSchema = partInfo.getPartitionSchema(); Map<String, Object> vals = new HashMap<String, Object>(); for (String fieldName : outputSchema.getFieldNames()) { if (dataSchema.getPosition(fieldName) == null) { // this entry of output is not present in the output schema // so, we first check the table schema to see if it is a part col if (partInfo.getPartitionValues().containsKey(fieldName)) { // First, get the appropriate field schema for this field HCatFieldSchema fschema = outputSchema.get(fieldName); // For a partition key type, this will be a primitive typeinfo. // Obtain relevant object inspector for this typeinfo ObjectInspector oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(fschema.getTypeInfo()); // get appropriate object from the string representation of the value in partInfo.getPartitionValues() // Essentially, partition values are represented as strings, but we want the actual object type associated Object objVal = ObjectInspectorConverters .getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi) .convert(partInfo.getPartitionValues().get(fieldName)); vals.put(fieldName, objVal); } else { vals.put(fieldName, null); } } } return vals; }
/** * gets values for fields requested by output schema which will not be in the data */ private static Map<String, Object> getColValsNotInDataColumns(HCatSchema outputSchema, PartInfo partInfo) throws HCatException { HCatSchema dataSchema = partInfo.getPartitionSchema(); Map<String, Object> vals = new HashMap<String, Object>(); for (String fieldName : outputSchema.getFieldNames()) { if (dataSchema.getPosition(fieldName) == null) { // this entry of output is not present in the output schema // so, we first check the table schema to see if it is a part col if (partInfo.getPartitionValues().containsKey(fieldName)) { // First, get the appropriate field schema for this field HCatFieldSchema fschema = outputSchema.get(fieldName); // For a partition key type, this will be a primitive typeinfo. // Obtain relevant object inspector for this typeinfo ObjectInspector oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(fschema.getTypeInfo()); // get appropriate object from the string representation of the value in partInfo.getPartitionValues() // Essentially, partition values are represented as strings, but we want the actual object type associated Object objVal = ObjectInspectorConverters .getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi) .convert(partInfo.getPartitionValues().get(fieldName)); vals.put(fieldName, objVal); } else { vals.put(fieldName, null); } } } return vals; }
/** * gets values for fields requested by output schema which will not be in the data */ private static Map<String, Object> getColValsNotInDataColumns(HCatSchema outputSchema, PartInfo partInfo) throws HCatException { HCatSchema dataSchema = partInfo.getPartitionSchema(); Map<String, Object> vals = new HashMap<String, Object>(); for (String fieldName : outputSchema.getFieldNames()) { if (dataSchema.getPosition(fieldName) == null) { // this entry of output is not present in the output schema // so, we first check the table schema to see if it is a part col if (partInfo.getPartitionValues().containsKey(fieldName)) { // First, get the appropriate field schema for this field HCatFieldSchema fschema = outputSchema.get(fieldName); // For a partition key type, this will be a primitive typeinfo. // Obtain relevant object inspector for this typeinfo ObjectInspector oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(fschema.getTypeInfo()); // get appropriate object from the string representation of the value in partInfo.getPartitionValues() // Essentially, partition values are represented as strings, but we want the actual object type associated Object objVal = ObjectInspectorConverters .getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi) .convert(partInfo.getPartitionValues().get(fieldName)); vals.put(fieldName, objVal); } else { vals.put(fieldName, null); } } } return vals; }
/** * gets values for fields requested by output schema which will not be in the data */ private static Map<String, Object> getColValsNotInDataColumns(HCatSchema outputSchema, PartInfo partInfo) throws HCatException { HCatSchema dataSchema = partInfo.getPartitionSchema(); Map<String, Object> vals = new HashMap<String, Object>(); for (String fieldName : outputSchema.getFieldNames()) { if (dataSchema.getPosition(fieldName) == null) { // this entry of output is not present in the output schema // so, we first check the table schema to see if it is a part col if (partInfo.getPartitionValues().containsKey(fieldName)) { // First, get the appropriate field schema for this field HCatFieldSchema fschema = outputSchema.get(fieldName); // For a partition key type, this will be a primitive typeinfo. // Obtain relevant object inspector for this typeinfo ObjectInspector oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(fschema.getTypeInfo()); // get appropriate object from the string representation of the value in partInfo.getPartitionValues() // Essentially, partition values are represented as strings, but we want the actual object type associated Object objVal = ObjectInspectorConverters .getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi) .convert(partInfo.getPartitionValues().get(fieldName)); vals.put(fieldName, objVal); } else { vals.put(fieldName, null); } } } return vals; }