public Integer getInt(String key, int iDef) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { Integer value = ((IntWritable) fieldValue).get(); return value; } } return iDef; }
@Override public Long getLong(String key, Long lDef) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { Long value = ((LongWritable) fieldValue).get(); return value; } } return lDef; }
@Override public String getString(String key, String strDef) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { String value = ((Text) fieldValue).toString(); return value; } } return strDef; }
@Override public Object get(String key) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; OrcStruct tmp = (OrcStruct) orcStruct.getFieldValue(key); if (tmp != null) { return new OrcRecord(tmp); } } return null; }
@Override public DataRecord getRecord(String key) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; OrcStruct tmp = (OrcStruct) orcStruct.getFieldValue(key); if (tmp != null) { return new OrcRecord(tmp); } } return null; }
@Override public Boolean getBoolean(String key, Boolean bDef) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { Boolean value = ((BooleanWritable) fieldValue).get(); return value; } } return bDef; }
public Float getFloat(String key, float fDef) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { Float value = ((FloatWritable) fieldValue).get(); return value; } } return fDef; }
@Override public Double getDouble(String key, Double dDef) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { Double value = ((DoubleWritable) fieldValue).get(); return value; } } return dDef; }
public Map<String, Object> getMap(String key) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; OrcMap value = (OrcMap) orcStruct.getFieldValue(key); if (value != null && value.size() > 0) { Map<String, Object> map = new HashMap<String, Object>(); Set<Map.Entry<Object, Object>> set = value.entrySet(); for (Map.Entry<Object, Object> entry : set) { if (entry.getValue() instanceof OrcStruct) { map.put(((Text) entry.getKey()).toString(), new OrcRecord(entry.getValue())); } else { map.put(((Text) entry.getKey()).toString(), entry.getValue()); } } return map; } } return null; }
static void setStructValue(TypeDescription schema, StructColumnVector vector, int row, OrcStruct value) { List<TypeDescription> children = schema.getChildren(); for(int c=0; c < value.getNumFields(); ++c) { setColumn(children.get(c), vector.fields[c], row, value.getFieldValue(c)); } }
@Override public Object[] getArray(String key) throws Exception { if (data instanceof OrcStruct) { OrcStruct orcStruct = (OrcStruct) data; WritableComparable fieldValue = orcStruct.getFieldValue(key); if (fieldValue != null) { Object[] value = ((OrcList) fieldValue).toArray(); if (value.length > 0) { if (!(value[0] instanceof OrcStruct)) { return value; } else { DataRecord[] dataRecords = new DataRecord[value.length]; for (int i = 0; i < value.length; i++) { dataRecords[i] = new OrcRecord(value[i]); } return dataRecords; } } } } return null; }
static OrcStruct nextStruct(ColumnVector vector, int row, TypeDescription schema, Object previous) { if (vector.isRepeating) { row = 0; } if (vector.noNulls || !vector.isNull[row]) { OrcStruct result; List<TypeDescription> childrenTypes = schema.getChildren(); int numChildren = childrenTypes.size(); if (previous == null || previous.getClass() != OrcStruct.class) { result = new OrcStruct(schema); } else { result = (OrcStruct) previous; } StructColumnVector struct = (StructColumnVector) vector; for(int f=0; f < numChildren; ++f) { result.setFieldValue(f, nextValue(struct.fields[f], row, childrenTypes.get(f), result.getFieldValue(f))); } return result; } else { return null; } }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!ensureBatch()) { return false; } if (schema.getCategory() == TypeDescription.Category.STRUCT) { OrcStruct result = (OrcStruct) row; List<TypeDescription> children = schema.getChildren(); int numberOfChildren = children.size(); for(int i=0; i < numberOfChildren; ++i) { result.setFieldValue(i, OrcMapredRecordReader.nextValue(batch.cols[i], rowInBatch, children.get(i), result.getFieldValue(i))); } } else { OrcMapredRecordReader.nextValue(batch.cols[0], rowInBatch, schema, row); } rowInBatch += 1; return true; }
@Override public boolean next(NullWritable key, V value) throws IOException { if (!ensureBatch()) { return false; } if (schema.getCategory() == TypeDescription.Category.STRUCT) { OrcStruct result = (OrcStruct) value; List<TypeDescription> children = schema.getChildren(); int numberOfChildren = children.size(); for(int i=0; i < numberOfChildren; ++i) { result.setFieldValue(i, nextValue(batch.cols[i], rowInBatch, children.get(i), result.getFieldValue(i))); } } else { nextValue(batch.cols[0], rowInBatch, schema, value); } rowInBatch += 1; return true; }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (key == null || key.fileName == null) { key = new CombineFileWritable(); key.fileName = path.toString(); } if (!ensureBatch()) { return false; } if (schema.getCategory() == TypeDescription.Category.STRUCT) { OrcStruct result = row; List<TypeDescription> children = schema.getChildren(); int numberOfChildren = children.size(); for (int i = 0; i < numberOfChildren; ++i) { result.setFieldValue(i, OrcMapredRecordReader.nextValue(batch.cols[i], rowInBatch, children.get(i), result.getFieldValue(i))); } } else { OrcMapredRecordReader.nextValue(batch.cols[0], rowInBatch, schema, row); } rowInBatch += 1; return true; }
@Override public void write(NullWritable nullWritable, V v) throws IOException { // if the batch is full, write it out. if (batch.size == batch.getMaxSize()) { writer.addRowBatch(batch); batch.reset(); } // add the new row int row = batch.size++; // skip over the OrcKey or OrcValue if (v instanceof OrcKey) { v = (V)((OrcKey) v).key; } else if (v instanceof OrcValue) { v = (V)((OrcValue) v).value; } if (isTopStruct) { for(int f=0; f < schema.getChildren().size(); ++f) { setColumn(schema.getChildren().get(f), batch.cols[f], row, ((OrcStruct) v).getFieldValue(f)); } } else { setColumn(schema, batch.cols[0], row, v); } }
@Override public void write(NullWritable nullWritable, V v) throws IOException { // if the batch is full, write it out. if (batch.size == batch.getMaxSize()) { writer.addRowBatch(batch); batch.reset(); } // add the new row int row = batch.size++; // skip over the OrcKey or OrcValue if (v instanceof OrcKey) { v = (V)((OrcKey) v).key; } else if (v instanceof OrcValue) { v = (V)((OrcValue) v).value; } if (isTopStruct) { for(int f=0; f < schema.getChildren().size(); ++f) { OrcMapredRecordWriter.setColumn(schema.getChildren().get(f), batch.cols[f], row, ((OrcStruct) v).getFieldValue(f)); } } else { OrcMapredRecordWriter.setColumn(schema, batch.cols[0], row, v); } }
public Row deserialize(OrcStruct orcStruct, Row reuse) { for (int i = 0; i < reuse.getArity(); i++) { reuse.setField(i, null); } for (int i = 0; i < columnIds.length; i++) { if (columnIds[i] != -1) { WritableComparable value = orcStruct.getFieldValue(columnIds[i]); if (value == null) { reuse.setField(i, null); } else { reuse.setField(i, fieldConverters[i].convert(value)); } } } return reuse; }