@Override public List<Entry<String, ByteString>> getSharedData() { return Collections.singletonList( new SimpleEntry<>( HIVE_ATTRIBUTE_KEY, ByteString.copyFrom(dataset.getReadDefinition().getExtendedProperty().asReadOnlyByteBuffer()) ) ); }
public static String[] getColumnsLowerCase(DatasetConfig datasetConfig) { final ByteString schemaBytes = DatasetHelper.getSchemaBytes(datasetConfig); if (schemaBytes != null) { Schema schema = Schema.getRootAsSchema(schemaBytes.asReadOnlyByteBuffer()); org.apache.arrow.vector.types.pojo.Schema s = org.apache.arrow.vector.types.pojo.Schema.convertSchema(schema); return s.getFields().stream().map(input -> input.getName().toLowerCase()).toArray(String[]::new); } else { // If virtual dataset was created with view fields if (datasetConfig.getType() == DatasetType.VIRTUAL_DATASET) { final List<ViewFieldType> viewFieldTypes = datasetConfig.getVirtualDataset().getSqlFieldsList(); if (notEmpty(viewFieldTypes)) { return viewFieldTypes.stream().map(input -> input.getName().toLowerCase()).toArray(String[]::new); } } } return new String[0]; }
private DatasetConfig update(DatasetConfig datasetConfig) { if (datasetConfig == null) { return null; } final io.protostuff.ByteString schemaBytes = DatasetHelper.getSchemaBytes(datasetConfig); if (schemaBytes == null) { return null; } try { OldSchema oldSchema = OldSchema.getRootAsOldSchema(schemaBytes.asReadOnlyByteBuffer()); byte[] newschemaBytes = convertFromOldSchema(oldSchema); datasetConfig.setRecordSchema(ByteString.copyFrom(newschemaBytes)); return datasetConfig; } catch (Exception e) { System.out.println("Unable to update Arrow Schema for: " + PathUtils .constructFullPath(Optional.ofNullable(datasetConfig.getFullPathList()).orElse(Lists.newArrayList()))); e.printStackTrace(System.out); return null; } }
public static BatchSchema deserialize(ByteString bytes) { Schema schema = Schema.getRootAsSchema(bytes.asReadOnlyByteBuffer()); org.apache.arrow.vector.types.pojo.Schema s = org.apache.arrow.vector.types.pojo.Schema.convertSchema(schema); return new BatchSchema(SelectionVectorMode.NONE, s.getFields()); }