private List<Map<String, Object>> project( List<Map<String,Object>> incomingRecords, List<String> projectCols) { List<Map<String,Object>> output = Lists.newArrayList(); for (Map<String, Object> incomingRecord : incomingRecords) { final JsonStringHashMap<String, Object> newRecord = new JsonStringHashMap<>(); for (String s : incomingRecord.keySet()) { if (projectCols.contains(s)) { newRecord.put(s, incomingRecord.get(s)); } } output.add(newRecord); } return output; }
/** * Convenience method to create a {@link JsonStringHashMap<String, Object> map} instance with the given key value sequence. * * Key value sequence consists of key - value pairs such that a key precedes its value. For instance: * * mapOf("name", "Adam", "age", 41) corresponds to {"name": "Adam", "age": 41} in JSON. */ public static JsonStringHashMap<String, Object> mapOf(Object... keyValueSequence) { Preconditions.checkArgument(keyValueSequence.length%2==0, "Length of key value sequence must be even"); final JsonStringHashMap<String, Object> map = new JsonStringHashMap<>(); for (int i=0; i<keyValueSequence.length; i+=2) { Object value = keyValueSequence[i+1]; if (value instanceof CharSequence) { value = new Text(value.toString()); } map.put((String)keyValueSequence[i], value); } return map; }
private List<JsonStringHashMap<String, Object>> flatten( List<JsonStringHashMap<String,Object>> incomingRecords, String colToFlatten, String flattenedDataColName) { List<JsonStringHashMap<String,Object>> output = Lists.newArrayList(); for (JsonStringHashMap<String, Object> incomingRecord : incomingRecords) { List<?> dataToFlatten = (List<?>) incomingRecord.get(colToFlatten); for (int i = 0; i < dataToFlatten.size(); i++) { final JsonStringHashMap<String, Object> newRecord = new JsonStringHashMap<>(); newRecord.put(flattenedDataColName, dataToFlatten.get(i)); for (String s : incomingRecord.keySet()) { if (s.equals(colToFlatten)) { continue; } newRecord.put(s, incomingRecord.get(s)); } output.add(newRecord); } } return output; }
private Object convertAvroValToDremio(Object value, boolean root) { if (value instanceof ByteBuffer) { ByteBuffer bb = ((ByteBuffer)value); byte[] val = new byte[((ByteBuffer)value).remaining()]; bb.get(val); bb.position(0); value = val; } else if (!root && value instanceof CharSequence) { value = new Text(value.toString()); } else if (value instanceof GenericData.Array) { GenericData.Array array = ((GenericData.Array) value); final JsonStringArrayList<Object> list = new JsonStringArrayList<>(); for (Object o : array) { list.add(convertAvroValToDremio(o, false)); } value = list; } else if (value instanceof GenericData.EnumSymbol) { value = value.toString(); } else if (value instanceof GenericData.Record) { GenericData.Record rec = ((GenericData.Record) value); final JsonStringHashMap<String, Object> newRecord = new JsonStringHashMap<>(); for (Schema.Field field : rec.getSchema().getFields()) { Object val = rec.get(field.name()); newRecord.put(field.name(), convertAvroValToDremio(val, false)); } value = newRecord; } return value; }